diff --git a/.github/aw/actions-lock.json b/.github/aw/actions-lock.json index 1f061b90..df05975d 100644 --- a/.github/aw/actions-lock.json +++ b/.github/aw/actions-lock.json @@ -5,6 +5,11 @@ "version": "v8", "sha": "ed597411d8f924073f98dfc5c65a23a2325f34cd" }, + "github/gh-aw-actions/setup@v0.81.6": { + "repo": "github/gh-aw-actions/setup", + "version": "v0.81.6", + "sha": "ba6380cc6e5be5d21677bebe04d52fb48e3abec7" + }, "githubnext/gh-aw/actions/setup@v0.36.0": { "repo": "githubnext/gh-aw/actions/setup", "version": "v0.36.0", diff --git a/.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py/SKILL.md index 04a975fe..98b28f5c 100644 --- a/.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py/SKILL.md @@ -347,7 +347,7 @@ if __name__ == "__main__": ## Best Practices -1. **This SDK is async-first** — use `async def` handlers and `async with` throughout. +1. **This SDK is async-first — use `async def` handlers and `async with` throughout.** 2. **Always use context managers for clients and async credentials.** Wrap every client in `with Client(...) as client:` (sync) or `async with Client(...) as client:` (async). For async `DefaultAzureCredential` from `azure.identity.aio`, also use `async with credential:` so tokens and transports are cleaned up. ## Reference Files diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/SKILL.md index f48d8a11..9caadcc9 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/SKILL.md @@ -240,3 +240,10 @@ request = AnalyzeTextOptions( 7. **Log analysis results** for audit and improvement 8. **Consider 8-severity mode** for finer-grained control 9. **Pre-moderate AI outputs** before showing to users + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/capabilities.md new file mode 100644 index 00000000..529c1776 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/capabilities.md @@ -0,0 +1,34 @@ +# azure-ai-contentsafety-py capability coverage + +**SDK/package**: `azure-ai-contentsafety` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Analyze Text` +- `Analyze Image` +- `Text Blocklist Management` +- `Severity Levels` + +## Non-hero scenarios + +- `Harm Categories`: | Category | Description | + See: [`non-hero-scenarios.md#harm-categories`](non-hero-scenarios.md#harm-categories) +- `Severity Scale`: | Level | Text Range | Image Range | Meaning | + See: [`non-hero-scenarios.md#severity-scale`](non-hero-scenarios.md#severity-scale) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..3d08cd45 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py/references/non-hero-scenarios.md @@ -0,0 +1,29 @@ +# azure-ai-contentsafety-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Harm Categories + +| Category | Description | +|----------|-------------| +| `Hate` | Attacks based on identity (race, religion, gender, etc.) | +| `Sexual` | Sexual content, relationships, anatomy | +| `Violence` | Physical harm, weapons, injury | +| `SelfHarm` | Self-injury, suicide, eating disorders | + +## Severity Scale + +| Level | Text Range | Image Range | Meaning | +|-------|------------|-------------|---------| +| 0 | Safe | Safe | No harmful content | +| 2 | Low | Low | Mild references | +| 4 | Medium | Medium | Moderate content | +| 6 | High | High | Severe content | + +## Client Types + +| Client | Purpose | +|--------|---------| +| `ContentSafetyClient` | Analyze text and images | +| `BlocklistClient` | Manage custom blocklists | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/SKILL.md index 1998fba3..5b59e23e 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/SKILL.md @@ -292,3 +292,10 @@ from azure.ai.contentunderstanding.models import ( 7. **Use async client** for high-throughput scenarios with `azure.identity.aio` credentials 8. **Handle long-running operations** — video/audio analysis can take minutes 9. **Use URL sources** when possible to avoid upload overhead + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/capabilities.md new file mode 100644 index 00000000..d03bbc13 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/capabilities.md @@ -0,0 +1,46 @@ +# azure-ai-contentunderstanding-py capability coverage + +**SDK/package**: `azure-ai-contentunderstanding` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Core Workflow` +- `Prebuilt Analyzers` +- `Analyze Document` +- `Access Document Content Details` + +## Non-hero scenarios + +- `Analyze Image`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#analyze-image`](non-hero-scenarios.md#analyze-image) +- `Analyze Video`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#analyze-video`](non-hero-scenarios.md#analyze-video) +- `Analyze Audio`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#analyze-audio`](non-hero-scenarios.md#analyze-audio) +- `Custom Analyzers`: Create custom analyzers with field schemas for specialized extraction: + See: [`non-hero-scenarios.md#custom-analyzers`](non-hero-scenarios.md#custom-analyzers) +- `Analyzer Management`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#analyzer-management`](non-hero-scenarios.md#analyzer-management) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Content Types`: | Class | For | Provides | + See: [`non-hero-scenarios.md#content-types`](non-hero-scenarios.md#content-types) +- `Model Imports`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#model-imports`](non-hero-scenarios.md#model-imports) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..f5a6a4fb --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py/references/non-hero-scenarios.md @@ -0,0 +1,175 @@ +# azure-ai-contentunderstanding-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Analyze Image + +```python +from azure.ai.contentunderstanding.models import AnalyzeInput + +poller = client.begin_analyze( + analyzer_id="prebuilt-imageSearch", + inputs=[AnalyzeInput(url="https://example.com/image.jpg")] +) +result = poller.result() +content = result.contents[0] +print(content.markdown) +``` + +## Analyze Video + +```python +from azure.ai.contentunderstanding.models import AnalyzeInput + +poller = client.begin_analyze( + analyzer_id="prebuilt-videoSearch", + inputs=[AnalyzeInput(url="https://example.com/video.mp4")] +) + +result = poller.result() + +# Access video content (AudioVisualContent) +content = result.contents[0] + +# Get transcript phrases with timing +for phrase in content.transcript_phrases: + print(f"[{phrase.start_time} - {phrase.end_time}]: {phrase.text}") + +# Get key frames (for video) +for frame in content.key_frames: + print(f"Frame at {frame.time}: {frame.description}") +``` + +## Analyze Audio + +```python +from azure.ai.contentunderstanding.models import AnalyzeInput + +poller = client.begin_analyze( + analyzer_id="prebuilt-audioSearch", + inputs=[AnalyzeInput(url="https://example.com/audio.mp3")] +) + +result = poller.result() + +# Access audio transcript +content = result.contents[0] +for phrase in content.transcript_phrases: + print(f"[{phrase.start_time}] {phrase.text}") +``` + +## Custom Analyzers + +Create custom analyzers with field schemas for specialized extraction: + +```python +# Create custom analyzer +analyzer = client.create_analyzer( + analyzer_id="my-invoice-analyzer", + analyzer={ + "description": "Custom invoice analyzer", + "base_analyzer_id": "prebuilt-documentSearch", + "field_schema": { + "fields": { + "vendor_name": {"type": "string"}, + "invoice_total": {"type": "number"}, + "line_items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": {"type": "string"}, + "amount": {"type": "number"} + } + } + } + } + } + } +) + +# Use custom analyzer +from azure.ai.contentunderstanding.models import AnalyzeInput + +poller = client.begin_analyze( + analyzer_id="my-invoice-analyzer", + inputs=[AnalyzeInput(url="https://example.com/invoice.pdf")] +) + +result = poller.result() + +# Access extracted fields +print(result.fields["vendor_name"]) +print(result.fields["invoice_total"]) +``` + +## Analyzer Management + +```python +# List all analyzers +analyzers = client.list_analyzers() +for analyzer in analyzers: + print(f"{analyzer.analyzer_id}: {analyzer.description}") + +# Get specific analyzer +analyzer = client.get_analyzer("prebuilt-documentSearch") + +# Delete custom analyzer +client.delete_analyzer("my-custom-analyzer") +``` + +## Async Client + +```python +import asyncio +import os +from azure.ai.contentunderstanding.aio import ContentUnderstandingClient +from azure.ai.contentunderstanding.models import AnalyzeInput +from azure.identity.aio import DefaultAzureCredential + +async def analyze_document(): + endpoint = os.environ["CONTENTUNDERSTANDING_ENDPOINT"] + async with DefaultAzureCredential() as credential: + async with ContentUnderstandingClient( + endpoint=endpoint, + credential=credential + ) as client: + poller = await client.begin_analyze( + analyzer_id="prebuilt-documentSearch", + inputs=[AnalyzeInput(url="https://example.com/doc.pdf")] + ) + result = await poller.result() + content = result.contents[0] + return content.markdown + +asyncio.run(analyze_document()) +``` + +## Content Types + +| Class | For | Provides | +|-------|-----|----------| +| `DocumentContent` | PDF, images, Office docs | Pages, tables, figures, paragraphs | +| `AudioVisualContent` | Audio, video files | Transcript phrases, timing, key frames | + +Both derive from `MediaContent` which provides basic info and markdown representation. + +## Model Imports + +```python +from azure.ai.contentunderstanding.models import ( + AnalyzeInput, + AnalyzeResult, + MediaContentKind, + DocumentContent, + AudioVisualContent, +) +``` + +## Client Types + +| Client | Purpose | +|--------|---------| +| `ContentUnderstandingClient` | Sync client for all operations | +| `ContentUnderstandingClient` (aio) | Async client for all operations | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/SKILL.md index 74a373cb..d875833e 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/SKILL.md @@ -96,4 +96,11 @@ with ConversationAnalysisClient(endpoint, credential) as client: } ) - print(f"Top intent: {result['result']['prediction']['topIntent']}") \ No newline at end of file + print(f"Top intent: {result['result']['prediction']['topIntent']}") + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/capabilities.md new file mode 100644 index 00000000..abaae4b5 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/capabilities.md @@ -0,0 +1,27 @@ +# azure-ai-language-conversations-py capability coverage + +**SDK/package**: `azure-ai-language-conversations` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Core workflow` + +## Non-hero scenarios + +- `Operational hardening`: Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. + See: [`non-hero-scenarios.md#operational-hardening`](non-hero-scenarios.md#operational-hardening) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..d7383045 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py/references/non-hero-scenarios.md @@ -0,0 +1,8 @@ +# azure-ai-language-conversations-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Operational hardening + +Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/SKILL.md index 64ae2381..c3b6d0e1 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/SKILL.md @@ -299,3 +299,10 @@ print(f"Default: {default_ds.name}") 7. **Register models** after successful training jobs 8. **Use pipelines** for multi-step workflows 9. **Tag resources** for organization and cost tracking + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/capabilities.md new file mode 100644 index 00000000..35896a80 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/capabilities.md @@ -0,0 +1,38 @@ +# azure-ai-ml-py capability coverage + +**SDK/package**: `azure-ai-ml` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Workspace Management` +- `Data Assets` +- `Model Registry` +- `Compute` + +## Non-hero scenarios + +- `Jobs`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#jobs`](non-hero-scenarios.md#jobs) +- `Pipelines`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#pipelines`](non-hero-scenarios.md#pipelines) +- `Environments`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#environments`](non-hero-scenarios.md#environments) +- `Datastores`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#datastores`](non-hero-scenarios.md#datastores) +- `MLClient Operations`: | Property | Operations | + See: [`non-hero-scenarios.md#mlclient-operations`](non-hero-scenarios.md#mlclient-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..f363f7b1 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-ml-py/references/non-hero-scenarios.md @@ -0,0 +1,104 @@ +# azure-ai-ml-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Jobs + +### Command Job + +```python +from azure.ai.ml import command, Input + +job = command( + code="./src", + command="python train.py --data ${{inputs.data}} --lr ${{inputs.learning_rate}}", + inputs={ + "data": Input(type="uri_folder", path="azureml:my-dataset:1"), + "learning_rate": 0.01 + }, + environment="AzureML-sklearn-1.0-ubuntu20.04-py38-cpu@latest", + compute="cpu-cluster", + display_name="training-job" +) + +returned_job = ml_client.jobs.create_or_update(job) +print(f"Job URL: {returned_job.studio_url}") +``` + +### Monitor Job + +```python +ml_client.jobs.stream(returned_job.name) +``` + +## Pipelines + +```python +from azure.ai.ml import dsl, Input, Output +from azure.ai.ml.entities import Pipeline + +@dsl.pipeline( + compute="cpu-cluster", + description="Training pipeline" +) +def training_pipeline(data_input): + prep_step = prep_component(data=data_input) + train_step = train_component( + data=prep_step.outputs.output_data, + learning_rate=0.01 + ) + return {"model": train_step.outputs.model} + +pipeline = training_pipeline( + data_input=Input(type="uri_folder", path="azureml:my-dataset:1") +) + +pipeline_job = ml_client.jobs.create_or_update(pipeline) +``` + +## Environments + +### Create Custom Environment + +```python +from azure.ai.ml.entities import Environment + +env = Environment( + name="my-env", + version="1", + image="mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04", + conda_file="./environment.yml" +) + +ml_client.environments.create_or_update(env) +``` + +## Datastores + +### List Datastores + +```python +for ds in ml_client.datastores.list(): + print(f"{ds.name}: {ds.type}") +``` + +### Get Default Datastore + +```python +default_ds = ml_client.datastores.get_default() +print(f"Default: {default_ds.name}") +``` + +## MLClient Operations + +| Property | Operations | +|----------|------------| +| `workspaces` | create, get, list, delete | +| `jobs` | create_or_update, get, list, stream, cancel | +| `models` | create_or_update, get, list, archive | +| `data` | create_or_update, get, list | +| `compute` | begin_create_or_update, get, list, delete | +| `environments` | create_or_update, get, list | +| `datastores` | create_or_update, get, list, get_default | +| `components` | create_or_update, get, list | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/SKILL.md index dc9d0d92..3f7ec0d3 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/SKILL.md @@ -252,3 +252,10 @@ async def analyze(): 5. **Use async client** for high-throughput scenarios 6. **Handle document errors** — results list may contain errors for some docs 7. **Specify language** when known to improve accuracy + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/capabilities.md new file mode 100644 index 00000000..19ce159a --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/capabilities.md @@ -0,0 +1,40 @@ +# azure-ai-textanalytics-py capability coverage + +**SDK/package**: `azure-ai-textanalytics` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Sentiment Analysis` +- `Entity Recognition` +- `PII Detection` +- `Key Phrase Extraction` + +## Non-hero scenarios + +- `Language Detection`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#language-detection`](non-hero-scenarios.md#language-detection) +- `Healthcare Text Analytics`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#healthcare-text-analytics`](non-hero-scenarios.md#healthcare-text-analytics) +- `Multiple Analysis (Batch)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#multiple-analysis-batch`](non-hero-scenarios.md#multiple-analysis-batch) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) +- `Available Operations`: | Method | Description | + See: [`non-hero-scenarios.md#available-operations`](non-hero-scenarios.md#available-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..461cb890 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py/references/non-hero-scenarios.md @@ -0,0 +1,104 @@ +# azure-ai-textanalytics-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Language Detection + +```python +documents = ["Ce document est en francais.", "This is written in English."] + +result = client.detect_language(documents) + +for doc in result: + if not doc.is_error: + print(f"Language: {doc.primary_language.name} ({doc.primary_language.iso6391_name})") + print(f"Confidence: {doc.primary_language.confidence_score:.2f}") +``` + +## Healthcare Text Analytics + +```python +documents = ["Patient has diabetes and was prescribed metformin 500mg twice daily."] + +poller = client.begin_analyze_healthcare_entities(documents) +result = poller.result() + +for doc in result: + if not doc.is_error: + for entity in doc.entities: + print(f"Entity: {entity.text}") + print(f" Category: {entity.category}") + print(f" Normalized: {entity.normalized_text}") + + # Entity links (UMLS, etc.) + for link in entity.data_sources: + print(f" Link: {link.name} - {link.entity_id}") +``` + +## Multiple Analysis (Batch) + +```python +from azure.ai.textanalytics import ( + RecognizeEntitiesAction, + ExtractKeyPhrasesAction, + AnalyzeSentimentAction +) + +documents = ["Microsoft announced new Azure AI features at Build conference."] + +poller = client.begin_analyze_actions( + documents, + actions=[ + RecognizeEntitiesAction(), + ExtractKeyPhrasesAction(), + AnalyzeSentimentAction() + ] +) + +results = poller.result() +for doc_results in results: + for result in doc_results: + if result.kind == "EntityRecognition": + print(f"Entities: {[e.text for e in result.entities]}") + elif result.kind == "KeyPhraseExtraction": + print(f"Key phrases: {result.key_phrases}") + elif result.kind == "SentimentAnalysis": + print(f"Sentiment: {result.sentiment}") +``` + +## Async Client + +```python +from azure.ai.textanalytics.aio import TextAnalyticsClient +from azure.identity.aio import DefaultAzureCredential + +async def analyze(): + async with DefaultAzureCredential() as credential: + async with TextAnalyticsClient( + endpoint=endpoint, + credential=credential + ) as client: + result = await client.analyze_sentiment(documents) + # Process results... +``` + +## Client Types + +| Client | Purpose | +|--------|---------| +| `TextAnalyticsClient` | All text analytics operations | +| `TextAnalyticsClient` (aio) | Async version | + +## Available Operations + +| Method | Description | +|--------|-------------| +| `analyze_sentiment` | Sentiment analysis with opinion mining | +| `recognize_entities` | Named entity recognition | +| `recognize_pii_entities` | PII detection and redaction | +| `recognize_linked_entities` | Entity linking to Wikipedia | +| `extract_key_phrases` | Key phrase extraction | +| `detect_language` | Language detection | +| `begin_analyze_healthcare_entities` | Healthcare NLP (long-running) | +| `begin_analyze_actions` | Multiple analyses in batch | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/SKILL.md index a6fd15e7..6f43ff43 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/SKILL.md @@ -24,12 +24,21 @@ pip install azure-ai-transcription ```bash TRANSCRIPTION_ENDPOINT=https://.cognitiveservices.azure.com -TRANSCRIPTION_KEY= +TRANSCRIPTION_KEY= # Required for all auth paths (this SDK currently uses key auth) ``` -## Authentication +## Authentication & Lifecycle -Use subscription key authentication (DefaultAzureCredential is not supported for this client): +> **🔑 Two rules apply to every code sample below:** +> +> 1. **This SDK currently requires API-key authentication.** `DefaultAzureCredential`/Entra ID isn't available for this client yet, so use `TRANSCRIPTION_KEY` from environment variables (never hardcoded in code). +> 2. **Wrap every client in a context manager** so HTTP transports and sockets are released deterministically: +> - Sync: `with (...) as client:` +> - Async: `async with (...) as client:` +> +> Snippets may abbreviate this setup, but production code should always follow both rules. + +Use subscription key authentication: ```python import os @@ -88,3 +97,10 @@ with TranscriptionClient( 6. **Specify language** to improve recognition accuracy 7. **Handle streaming backpressure** for real-time transcription 8. **Close transcription sessions** when complete + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/capabilities.md new file mode 100644 index 00000000..9f35147f --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/capabilities.md @@ -0,0 +1,28 @@ +# azure-ai-transcription-py capability coverage + +**SDK/package**: `azure-ai-transcription` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Transcription (Batch)` +- `Transcription (Real-time)` + +## Non-hero scenarios + +- `Operational hardening`: Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. + See: [`non-hero-scenarios.md#operational-hardening`](non-hero-scenarios.md#operational-hardening) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..a1532705 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py/references/non-hero-scenarios.md @@ -0,0 +1,8 @@ +# azure-ai-transcription-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Operational hardening + +Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/SKILL.md index 07379f43..4cf2eca9 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/SKILL.md @@ -276,3 +276,10 @@ async def translate_documents(): 7. **Separate target containers** for each language 8. **Use async client** for multiple concurrent jobs 9. **Check supported formats** before submitting documents + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/capabilities.md new file mode 100644 index 00000000..d0ead4f4 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/capabilities.md @@ -0,0 +1,44 @@ +# azure-ai-translation-document-py capability coverage + +**SDK/package**: `azure-ai-translation-document` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Basic Document Translation` +- `Multiple Target Languages` +- `Translate Single Document` +- `Check Translation Status` + +## Non-hero scenarios + +- `List Document Statuses`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-document-statuses`](non-hero-scenarios.md#list-document-statuses) +- `Cancel Translation`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#cancel-translation`](non-hero-scenarios.md#cancel-translation) +- `Using Glossary`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#using-glossary`](non-hero-scenarios.md#using-glossary) +- `Supported Document Formats`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#supported-document-formats`](non-hero-scenarios.md#supported-document-formats) +- `Supported Languages`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#supported-languages`](non-hero-scenarios.md#supported-languages) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Supported Formats`: | Category | Formats | + See: [`non-hero-scenarios.md#supported-formats`](non-hero-scenarios.md#supported-formats) +- `Storage Requirements`: - Source and target containers must be Azure Blob Storage + See: [`non-hero-scenarios.md#storage-requirements`](non-hero-scenarios.md#storage-requirements) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..cf9cb490 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py/references/non-hero-scenarios.md @@ -0,0 +1,105 @@ +# azure-ai-translation-document-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## List Document Statuses + +```python +# Get status of individual documents in a job +operation_id = poller.id +document_statuses = client.list_document_statuses(operation_id) + +for doc in document_statuses: + print(f"Document: {doc.source_document_url}") + print(f" Status: {doc.status}") + print(f" Translated to: {doc.translated_to}") + if doc.error: + print(f" Error: {doc.error.message}") +``` + +## Cancel Translation + +```python +# Cancel a running translation +client.cancel_translation(operation_id) +``` + +## Using Glossary + +```python +from azure.ai.translation.document import TranslationGlossary + +poller = client.begin_translation( + inputs=[ + DocumentTranslationInput( + source_url=source_url, + targets=[ + TranslationTarget( + target_url=target_url, + language="es", + glossaries=[ + TranslationGlossary( + glossary_url="https://.blob.core.windows.net/glossary/terms.csv?", + file_format="csv" + ) + ] + ) + ] + ) + ] +) +``` + +## Supported Document Formats + +```python +# Get supported formats +formats = client.get_supported_document_formats() + +for fmt in formats: + print(f"Format: {fmt.format}") + print(f" Extensions: {fmt.file_extensions}") + print(f" Content types: {fmt.content_types}") +``` + +## Supported Languages + +```python +# Get supported languages +languages = client.get_supported_languages() + +for lang in languages: + print(f"Language: {lang.name} ({lang.code})") +``` + +## Async Client + +```python +from azure.ai.translation.document.aio import DocumentTranslationClient +from azure.identity.aio import DefaultAzureCredential + +async def translate_documents(): + async with DefaultAzureCredential() as credential: + async with DocumentTranslationClient( + endpoint=endpoint, + credential=credential, + ) as client: + poller = await client.begin_translation(inputs=[...]) + result = await poller.result() +``` + +## Supported Formats + +| Category | Formats | +|----------|---------| +| Documents | DOCX, PDF, PPTX, XLSX, HTML, TXT, RTF | +| Structured | CSV, TSV, JSON, XML | +| Localization | XLIFF, XLF, MHTML | + +## Storage Requirements + +- Source and target containers must be Azure Blob Storage +- Use SAS tokens with appropriate permissions: + - Source: Read, List + - Target: Write, List diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/SKILL.md index 11d6f6fc..e8ed670c 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/SKILL.md @@ -298,3 +298,10 @@ async def translate_text(): 7. **Handle profanity** appropriately for your application 8. **Use html text_type** when translating HTML content 9. **Include alignment** for applications needing word mapping + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/capabilities.md new file mode 100644 index 00000000..379de29c --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/capabilities.md @@ -0,0 +1,44 @@ +# azure-ai-translation-text-py capability coverage + +**SDK/package**: `azure-ai-translation-text` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Basic Translation` +- `Translate to Multiple Languages` +- `Specify Source Language` +- `Language Detection` + +## Non-hero scenarios + +- `Transliteration`: Convert text from one script to another: + See: [`non-hero-scenarios.md#transliteration`](non-hero-scenarios.md#transliteration) +- `Dictionary Lookup`: Find alternate translations and definitions: + See: [`non-hero-scenarios.md#dictionary-lookup`](non-hero-scenarios.md#dictionary-lookup) +- `Dictionary Examples`: Get usage examples for translations: + See: [`non-hero-scenarios.md#dictionary-examples`](non-hero-scenarios.md#dictionary-examples) +- `Get Supported Languages`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#get-supported-languages`](non-hero-scenarios.md#get-supported-languages) +- `Break Sentence`: Identify sentence boundaries: + See: [`non-hero-scenarios.md#break-sentence`](non-hero-scenarios.md#break-sentence) +- `Translation Options`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#translation-options`](non-hero-scenarios.md#translation-options) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Client Methods`: | Method | Description | + See: [`non-hero-scenarios.md#client-methods`](non-hero-scenarios.md#client-methods) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..c2b870e2 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py/references/non-hero-scenarios.md @@ -0,0 +1,150 @@ +# azure-ai-translation-text-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Transliteration + +Convert text from one script to another: + +```python +result = client.transliterate( + body=["konnichiwa"], + language="ja", + from_script="Latn", # From Latin script + to_script="Jpan" # To Japanese script +) + +for item in result: + print(f"Transliterated: {item.text}") + print(f"Script: {item.script}") +``` + +## Dictionary Lookup + +Find alternate translations and definitions: + +```python +result = client.lookup_dictionary_entries( + body=["fly"], + from_parameter="en", + to="es" +) + +for item in result: + print(f"Source: {item.normalized_source} ({item.display_source})") + for translation in item.translations: + print(f" Translation: {translation.normalized_target}") + print(f" Part of speech: {translation.pos_tag}") + print(f" Confidence: {translation.confidence:.2f}") +``` + +## Dictionary Examples + +Get usage examples for translations: + +```python +from azure.ai.translation.text.models import DictionaryExampleTextItem + +result = client.lookup_dictionary_examples( + body=[DictionaryExampleTextItem(text="fly", translation="volar")], + from_parameter="en", + to="es" +) + +for item in result: + for example in item.examples: + print(f"Source: {example.source_prefix}{example.source_term}{example.source_suffix}") + print(f"Target: {example.target_prefix}{example.target_term}{example.target_suffix}") +``` + +## Get Supported Languages + +```python +# Get all supported languages +languages = client.get_supported_languages() + +# Translation languages +print("Translation languages:") +for code, lang in languages.translation.items(): + print(f" {code}: {lang.name} ({lang.native_name})") + +# Transliteration languages +print("\nTransliteration languages:") +for code, lang in languages.transliteration.items(): + print(f" {code}: {lang.name}") + for script in lang.scripts: + print(f" {script.code} -> {[t.code for t in script.to_scripts]}") + +# Dictionary languages +print("\nDictionary languages:") +for code, lang in languages.dictionary.items(): + print(f" {code}: {lang.name}") +``` + +## Break Sentence + +Identify sentence boundaries: + +```python +result = client.find_sentence_boundaries( + body=["Hello! How are you? I hope you are well."], + language="en" +) + +for item in result: + print(f"Sentence lengths: {item.sent_len}") +``` + +## Translation Options + +```python +result = client.translate( + body=["Hello, world!"], + to=["de"], + text_type="html", # "plain" or "html" + profanity_action="Marked", # "NoAction", "Deleted", "Marked" + profanity_marker="Asterisk", # "Asterisk", "Tag" + include_alignment=True, # Include word alignment + include_sentence_length=True # Include sentence boundaries +) + +for item in result: + translation = item.translations[0] + print(f"Translated: {translation.text}") + if translation.alignment: + print(f"Alignment: {translation.alignment.proj}") + if translation.sent_len: + print(f"Sentence lengths: {translation.sent_len.src_sent_len}") +``` + +## Async Client + +```python +from azure.ai.translation.text.aio import TextTranslationClient +from azure.identity.aio import DefaultAzureCredential + +async def translate_text(): + async with DefaultAzureCredential() as credential: + async with TextTranslationClient( + credential=credential, + endpoint=endpoint, + ) as client: + result = await client.translate( + body=["Hello, world!"], + to=["es"] + ) + print(result[0].translations[0].text) +``` + +## Client Methods + +| Method | Description | +|--------|-------------| +| `translate` | Translate text to one or more languages | +| `transliterate` | Convert text between scripts | +| `detect` | Detect language of text | +| `find_sentence_boundaries` | Identify sentence boundaries | +| `lookup_dictionary_entries` | Dictionary lookup for translations | +| `lookup_dictionary_examples` | Get usage examples | +| `get_supported_languages` | List supported languages | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/SKILL.md index ba1393b7..adaaf3c0 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/SKILL.md @@ -291,3 +291,10 @@ except HttpResponseError as e: 7. **Specify language** for localized captions 8. **Use smart_crops_aspect_ratios** matching your thumbnail requirements 9. **Cache results** when analyzing the same image multiple times + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/capabilities.md new file mode 100644 index 00000000..b24da438 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/capabilities.md @@ -0,0 +1,46 @@ +# azure-ai-vision-imageanalysis-py capability coverage + +**SDK/package**: `azure-ai-vision-imageanalysis` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Analyze Image from URL` +- `Analyze Image from File` +- `Image Caption` +- `Dense Captions (Multiple Regions)` + +## Non-hero scenarios + +- `Tags`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#tags`](non-hero-scenarios.md#tags) +- `Object Detection`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#object-detection`](non-hero-scenarios.md#object-detection) +- `OCR (Text Extraction)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#ocr-text-extraction`](non-hero-scenarios.md#ocr-text-extraction) +- `People Detection`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#people-detection`](non-hero-scenarios.md#people-detection) +- `Smart Cropping`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#smart-cropping`](non-hero-scenarios.md#smart-cropping) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Visual Features`: | Feature | Description | + See: [`non-hero-scenarios.md#visual-features`](non-hero-scenarios.md#visual-features) +- `Error Handling`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#error-handling`](non-hero-scenarios.md#error-handling) +- `Image Requirements`: - Formats: JPEG, PNG, GIF, BMP, WEBP, ICO, TIFF, MPO + See: [`non-hero-scenarios.md#image-requirements`](non-hero-scenarios.md#image-requirements) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..fe519431 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py/references/non-hero-scenarios.md @@ -0,0 +1,137 @@ +# azure-ai-vision-imageanalysis-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Tags + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.TAGS] +) + +if result.tags: + for tag in result.tags.list: + print(f"Tag: {tag.name} (confidence: {tag.confidence:.2f})") +``` + +## Object Detection + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.OBJECTS] +) + +if result.objects: + for obj in result.objects.list: + print(f"Object: {obj.tags[0].name}") + print(f" Confidence: {obj.tags[0].confidence:.2f}") + box = obj.bounding_box + print(f" Bounding box: x={box.x}, y={box.y}, w={box.width}, h={box.height}") +``` + +## OCR (Text Extraction) + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.READ] +) + +if result.read: + for block in result.read.blocks: + for line in block.lines: + print(f"Line: {line.text}") + print(f" Bounding polygon: {line.bounding_polygon}") + + # Word-level details + for word in line.words: + print(f" Word: {word.text} (confidence: {word.confidence:.2f})") +``` + +## People Detection + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.PEOPLE] +) + +if result.people: + for person in result.people.list: + print(f"Person detected:") + print(f" Confidence: {person.confidence:.2f}") + box = person.bounding_box + print(f" Bounding box: x={box.x}, y={box.y}, w={box.width}, h={box.height}") +``` + +## Smart Cropping + +```python +result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.SMART_CROPS], + smart_crops_aspect_ratios=[0.9, 1.33, 1.78] # Portrait, 4:3, 16:9 +) + +if result.smart_crops: + for crop in result.smart_crops.list: + print(f"Aspect ratio: {crop.aspect_ratio}") + box = crop.bounding_box + print(f" Crop region: x={box.x}, y={box.y}, w={box.width}, h={box.height}") +``` + +## Async Client + +```python +from azure.ai.vision.imageanalysis.aio import ImageAnalysisClient +from azure.identity.aio import DefaultAzureCredential + +async def analyze_image(): + async with DefaultAzureCredential() as credential: + async with ImageAnalysisClient( + endpoint=endpoint, + credential=credential + ) as client: + result = await client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.CAPTION] + ) + print(result.caption.text) +``` + +## Visual Features + +| Feature | Description | +|---------|-------------| +| `CAPTION` | Single sentence describing the image | +| `DENSE_CAPTIONS` | Captions for multiple regions | +| `TAGS` | Content tags (objects, scenes, actions) | +| `OBJECTS` | Object detection with bounding boxes | +| `READ` | OCR text extraction | +| `PEOPLE` | People detection with bounding boxes | +| `SMART_CROPS` | Suggested crop regions for thumbnails | + +## Error Handling + +```python +from azure.core.exceptions import HttpResponseError + +try: + result = client.analyze_from_url( + image_url=image_url, + visual_features=[VisualFeatures.CAPTION] + ) +except HttpResponseError as e: + print(f"Status code: {e.status_code}") + print(f"Reason: {e.reason}") + print(f"Message: {e.error.message}") +``` + +## Image Requirements + +- Formats: JPEG, PNG, GIF, BMP, WEBP, ICO, TIFF, MPO +- Max size: 20 MB +- Dimensions: 50x50 to 16000x16000 pixels diff --git a/.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py/SKILL.md index 1a3f3aa1..512d0686 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py/SKILL.md @@ -329,7 +329,7 @@ except ConnectionError as e: ## Best Practices -1. **This SDK is async-only; use `azure.ai.voicelive.aio` throughout.** Do not try to pair it with sync clients from other Azure SDKs in the same call path — keep the whole request path async. +1. **This SDK is async-only; use the `.aio` namespace throughout.** Do not try to pair it with sync clients from other Azure SDKs in the same call path — keep the whole request path async. 2. **Always use context managers for clients and async credentials.** Wrap every connection in `async with connect(...) as conn:`. For async `DefaultAzureCredential` from `azure.identity.aio`, also use `async with credential:` so tokens and transports are cleaned up. ## References diff --git a/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/SKILL.md index 9fe7feb9..bd07f043 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/SKILL.md @@ -256,3 +256,10 @@ async def main(): 7. **Use Entra ID** instead of connection strings in production 8. **Refresh settings periodically** in long-running applications 9. **Use feature flags** for gradual rollouts and A/B testing + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/capabilities.md new file mode 100644 index 00000000..9b454b1c --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/capabilities.md @@ -0,0 +1,34 @@ +# azure-appconfiguration-py capability coverage + +**SDK/package**: `azure-appconfiguration` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Configuration Settings` +- `List Settings` +- `Feature Flags` +- `Read-Only Settings` + +## Non-hero scenarios + +- `Snapshots`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#snapshots`](non-hero-scenarios.md#snapshots) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Client Operations`: | Operation | Description | + See: [`non-hero-scenarios.md#client-operations`](non-hero-scenarios.md#client-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..fe023531 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py/references/non-hero-scenarios.md @@ -0,0 +1,60 @@ +# azure-appconfiguration-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Snapshots + +### Create Snapshot + +```python +from azure.appconfiguration import ConfigurationSnapshot, ConfigurationSettingFilter + +snapshot = ConfigurationSnapshot( + name="v1-snapshot", + filters=[ + ConfigurationSettingFilter(key="app:*", label="production") + ] +) + +created = client.begin_create_snapshot( + name="v1-snapshot", + snapshot=snapshot +).result() +``` + +### List Snapshot Settings + +```python +settings = client.list_configuration_settings( + snapshot_name="v1-snapshot" +) +``` + +## Async Client + +```python +from azure.appconfiguration.aio import AzureAppConfigurationClient +from azure.identity.aio import DefaultAzureCredential + +async def main(): + async with DefaultAzureCredential() as credential: + async with AzureAppConfigurationClient( + base_url=endpoint, + credential=credential + ) as client: + setting = await client.get_configuration_setting(key="app:message") + print(setting.value) +``` + +## Client Operations + +| Operation | Description | +|-----------|-------------| +| `get_configuration_setting` | Get single setting | +| `set_configuration_setting` | Create or update setting | +| `delete_configuration_setting` | Delete setting | +| `list_configuration_settings` | List with filters | +| `set_read_only` | Lock/unlock setting | +| `begin_create_snapshot` | Create point-in-time snapshot | +| `list_snapshots` | List all snapshots | diff --git a/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/SKILL.md index e1a8305c..fb6ab982 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/SKILL.md @@ -273,3 +273,10 @@ for manifest in client.list_manifest_properties("my-image"): 7. **Use async client** for high-throughput operations 8. **Order by last_updated** to find recent/old images 9. **Check manifest.tags** before deleting to avoid removing tagged images + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/capabilities.md new file mode 100644 index 00000000..bea1d77b --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/capabilities.md @@ -0,0 +1,38 @@ +# azure-containerregistry-py capability coverage + +**SDK/package**: `azure-containerregistry` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `List Repositories` +- `Repository Operations` +- `List Tags` +- `Manifest Operations` + +## Non-hero scenarios + +- `Tag Operations`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#tag-operations`](non-hero-scenarios.md#tag-operations) +- `Upload and Download Artifacts`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#upload-and-download-artifacts`](non-hero-scenarios.md#upload-and-download-artifacts) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Clean Up Old Images`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#clean-up-old-images`](non-hero-scenarios.md#clean-up-old-images) +- `Client Operations`: | Operation | Description | + See: [`non-hero-scenarios.md#client-operations`](non-hero-scenarios.md#client-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..daa07c57 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-containerregistry-py/references/non-hero-scenarios.md @@ -0,0 +1,80 @@ +# azure-containerregistry-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Tag Operations + +### Get Tag Properties + +```python +tag = client.get_tag_properties("my-image", "latest") +print(f"Digest: {tag.digest}") +print(f"Created: {tag.created_on}") +``` + +### Delete Tag + +```python +client.delete_tag("my-image", "old-tag") +``` + +## Upload and Download Artifacts + +```python +from azure.containerregistry import ContainerRegistryClient + +with ContainerRegistryClient(endpoint, DefaultAzureCredential()) as client: + # Download manifest + manifest = client.download_manifest("my-image", "latest") + print(f"Media type: {manifest.media_type}") + print(f"Digest: {manifest.digest}") + + # Download blob + blob = client.download_blob("my-image", "sha256:abc123...") + with open("layer.tar.gz", "wb") as f: + for chunk in blob: + f.write(chunk) +``` + +## Async Client + +```python +from azure.containerregistry.aio import ContainerRegistryClient +from azure.identity.aio import DefaultAzureCredential + +async def list_repos(): + async with DefaultAzureCredential() as credential: + async with ContainerRegistryClient(endpoint, credential) as client: + async for repo in client.list_repository_names(): + print(repo) +``` + +## Clean Up Old Images + +```python +from datetime import datetime, timedelta, timezone + +cutoff = datetime.now(timezone.utc) - timedelta(days=30) + +for manifest in client.list_manifest_properties("my-image"): + if manifest.last_updated_on < cutoff and not manifest.tags: + print(f"Deleting {manifest.digest}") + client.delete_manifest("my-image", manifest.digest) +``` + +## Client Operations + +| Operation | Description | +|-----------|-------------| +| `list_repository_names` | List all repositories | +| `get_repository_properties` | Get repository metadata | +| `delete_repository` | Delete repository and all images | +| `list_tag_properties` | List tags in repository | +| `get_tag_properties` | Get tag metadata | +| `delete_tag` | Delete specific tag | +| `list_manifest_properties` | List manifests in repository | +| `get_manifest_properties` | Get manifest metadata | +| `delete_manifest` | Delete manifest by digest | +| `download_manifest` | Download manifest content | +| `download_blob` | Download layer blob | diff --git a/.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py/SKILL.md index 9fcb9385..66973512 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py/SKILL.md @@ -223,7 +223,7 @@ async def test_get_project_by_id_returns_project(mock_cosmos_container): ## Best Practices -1. **This skill uses async throughout (`azure.cosmos.aio`); do not mix with the sync `azure.cosmos` client.** Keep the whole FastAPI request path async — don't pair sync Cosmos calls with async handlers. +1. **Pick sync OR async and stay consistent.** Do not mix `azure.xxx` sync clients with `azure.xxx.aio` async clients in the same call path. Choose one mode per module. 2. **Always use context managers for clients and async credentials.** Wrap the client in `async with CosmosClient(...) as client:` (or manage its lifetime via FastAPI lifespan and close it explicitly). For async `DefaultAzureCredential` from `azure.identity.aio`, also use `async with credential:` so tokens and transports are cleaned up. ## Reference Files diff --git a/.github/plugins/azure-sdk-python/skills/azure-cosmos-py/scripts/setup_cosmos_container.py b/.github/plugins/azure-sdk-python/skills/azure-cosmos-py/scripts/setup_cosmos_container.py index 88cb5d06..d9baedf1 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-cosmos-py/scripts/setup_cosmos_container.py +++ b/.github/plugins/azure-sdk-python/skills/azure-cosmos-py/scripts/setup_cosmos_container.py @@ -31,7 +31,7 @@ def get_cosmos_client() -> CosmosClient: endpoint = os.environ.get("COSMOS_ENDPOINT") if not endpoint: raise ValueError("COSMOS_ENDPOINT environment variable required") - + # Try key auth first, fall back to DefaultAzureCredential key = os.environ.get("COSMOS_KEY") if key: @@ -44,33 +44,33 @@ def get_cosmos_client() -> CosmosClient: def create_indexing_policy( include_paths: list[str] | None = None, exclude_paths: list[str] | None = None, - composite_indexes: list[list[dict]] | None = None + composite_indexes: list[list[dict]] | None = None, ) -> dict[str, Any]: """Build an indexing policy.""" policy = { "indexingMode": "consistent", "automatic": True, "includedPaths": [], - "excludedPaths": [] + "excludedPaths": [], } - + # Include paths (default: all) if include_paths: policy["includedPaths"] = [{"path": p} for p in include_paths] else: policy["includedPaths"] = [{"path": "/*"}] - + # Exclude paths if exclude_paths: policy["excludedPaths"] = [{"path": p} for p in exclude_paths] - + # Always exclude _etag - policy["excludedPaths"].append({"path": "/_etag/?")}) - + policy["excludedPaths"].append({"path": "/_etag/?"}) + # Composite indexes for ORDER BY on multiple fields if composite_indexes: policy["compositeIndexes"] = composite_indexes - + return policy @@ -81,10 +81,10 @@ def create_container( partition_key_paths: list[str], throughput: int | None = None, ttl: int | None = None, - indexing_policy: dict | None = None + indexing_policy: dict | None = None, ) -> dict[str, Any]: """Create or update a Cosmos DB container.""" - + # Get or create database try: database = client.create_database_if_not_exists(id=database_id) @@ -92,41 +92,37 @@ def create_container( except CosmosHttpResponseError as e: print(f"Error creating database: {e.message}") raise - + # Build partition key if len(partition_key_paths) == 1: partition_key = PartitionKey(path=partition_key_paths[0]) else: # Hierarchical partition key partition_key = PartitionKey(path=partition_key_paths) - + # Container properties - container_props = { - "id": container_id, - "partition_key": partition_key - } - + container_props = {"id": container_id, "partition_key": partition_key} + # Add TTL if specified if ttl is not None: container_props["default_time_to_live"] = ttl - + # Add indexing policy if specified if indexing_policy: container_props["indexing_policy"] = indexing_policy - + # Create container try: if throughput: container = database.create_container_if_not_exists( - **container_props, - offer_throughput=throughput + **container_props, offer_throughput=throughput ) else: container = database.create_container_if_not_exists(**container_props) - + print(f"Container: {container_id}") print(f"Partition key: {partition_key_paths}") - + except CosmosHttpResponseError as e: if e.status_code == 409: print(f"Container {container_id} already exists") @@ -134,16 +130,16 @@ def create_container( else: print(f"Error creating container: {e.message}") raise - + # Get container properties properties = container.read() - + return { "database": database_id, "container": container_id, "partition_key": partition_key_paths, "self_link": properties.get("_self"), - "resource_id": properties.get("_rid") + "resource_id": properties.get("_rid"), } @@ -151,14 +147,14 @@ def show_container_info(client: CosmosClient, database_id: str, container_id: st """Display detailed container information.""" database = client.get_database_client(database_id) container = database.get_container_client(container_id) - + properties = container.read() - + print("\n=== Container Information ===") print(f"Database: {database_id}") print(f"Container: {container_id}") print(f"Partition Key: {properties.get('partitionKey', {}).get('paths', [])}") - + # TTL ttl = properties.get("defaultTtl") if ttl == -1: @@ -167,25 +163,29 @@ def show_container_info(client: CosmosClient, database_id: str, container_id: st print(f"TTL: {ttl} seconds") else: print("TTL: Disabled") - + # Indexing policy index_policy = properties.get("indexingPolicy", {}) print(f"Indexing Mode: {index_policy.get('indexingMode', 'consistent')}") - + # Throughput try: offer = container.read_offer() print(f"Throughput: {offer.offer_throughput} RU/s") if offer.properties.get("content", {}).get("offerAutopilotSettings"): - max_throughput = offer.properties["content"]["offerAutopilotSettings"]["maxThroughput"] + max_throughput = offer.properties["content"]["offerAutopilotSettings"][ + "maxThroughput" + ] print(f"Autoscale Max: {max_throughput} RU/s") except Exception: print("Throughput: Serverless or database-level") - + # Item count (approximate) try: query = "SELECT VALUE COUNT(1) FROM c" - count = list(container.query_items(query=query, enable_cross_partition_query=True))[0] + count = list( + container.query_items(query=query, enable_cross_partition_query=True) + )[0] print(f"Item Count: ~{count}") except Exception: print("Item Count: Unable to retrieve") @@ -195,84 +195,77 @@ def main(): parser = argparse.ArgumentParser( description="Create and configure Cosmos DB containers", formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=__doc__ - ) - - parser.add_argument( - "--database", "-d", - required=True, - help="Database ID" - ) - parser.add_argument( - "--container", "-c", - required=True, - help="Container ID" + epilog=__doc__, ) + + parser.add_argument("--database", "-d", required=True, help="Database ID") + parser.add_argument("--container", "-c", required=True, help="Container ID") parser.add_argument( - "--partition-key", "-pk", + "--partition-key", + "-pk", nargs="+", required=True, - help="Partition key path(s). Use multiple for hierarchical keys." + help="Partition key path(s). Use multiple for hierarchical keys.", ) parser.add_argument( - "--throughput", "-t", + "--throughput", + "-t", type=int, - help="Provisioned throughput in RU/s (omit for serverless)" + help="Provisioned throughput in RU/s (omit for serverless)", ) parser.add_argument( "--serverless", action="store_true", - help="Use serverless mode (no throughput provisioning)" + help="Use serverless mode (no throughput provisioning)", ) parser.add_argument( - "--ttl", - type=int, - help="Default TTL in seconds (-1 for per-item TTL)" + "--ttl", type=int, help="Default TTL in seconds (-1 for per-item TTL)" ) parser.add_argument( "--exclude-paths", nargs="+", - help="Paths to exclude from indexing (e.g., /large_field/*)" + help="Paths to exclude from indexing (e.g., /large_field/*)", ) parser.add_argument( "--composite-index", action="append", nargs="+", metavar="PATH", - help="Composite index paths (can specify multiple times). Format: --composite-index /field1 /field2" + help="Composite index paths (can specify multiple times). Format: --composite-index /field1 /field2", ) parser.add_argument( "--info", action="store_true", - help="Show container information instead of creating" + help="Show container information instead of creating", ) parser.add_argument( - "--output", "-o", + "--output", + "-o", choices=["json", "text"], default="text", - help="Output format (default: text)" + help="Output format (default: text)", ) - + args = parser.parse_args() - + # Validate arguments if args.throughput and args.serverless: print("Error: Cannot specify both --throughput and --serverless") sys.exit(1) - + # Ensure partition key paths start with / partition_keys = [] for pk in args.partition_key: if not pk.startswith("/"): pk = f"/{pk}" partition_keys.append(pk) - + try: client = get_cosmos_client() except ValueError as e: print(f"Error: {e}") sys.exit(1) - + # Show info mode if args.info: try: @@ -281,7 +274,7 @@ def main(): print(f"Error: {e.message}") sys.exit(1) return - + # Build indexing policy indexing_policy = None if args.exclude_paths or args.composite_index: @@ -291,12 +284,11 @@ def main(): [{"path": p, "order": "ascending"} for p in index_paths] for index_paths in args.composite_index ] - + indexing_policy = create_indexing_policy( - exclude_paths=args.exclude_paths, - composite_indexes=composite_indexes + exclude_paths=args.exclude_paths, composite_indexes=composite_indexes ) - + # Create container try: result = create_container( @@ -306,12 +298,12 @@ def main(): partition_key_paths=partition_keys, throughput=args.throughput if not args.serverless else None, ttl=args.ttl, - indexing_policy=indexing_policy + indexing_policy=indexing_policy, ) except CosmosHttpResponseError as e: print(f"Error: {e.message}") sys.exit(1) - + # Output result if args.output == "json": print(json.dumps(result, indent=2)) @@ -326,7 +318,7 @@ def main(): print("Throughput: Serverless") if args.ttl: print(f"TTL: {args.ttl} seconds") - + print("\nContainer ready for use!") diff --git a/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/SKILL.md index ef513ce5..6376581b 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/SKILL.md @@ -269,3 +269,10 @@ asyncio.run(table_operations()) 8. **Use parameterized queries** to prevent injection 9. **Keep entities small** — max 1MB per entity 10. **Use async client** for high-throughput scenarios + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/capabilities.md new file mode 100644 index 00000000..efb547ee --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/capabilities.md @@ -0,0 +1,34 @@ +# azure-data-tables-py capability coverage + +**SDK/package**: `azure-data-tables` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Client Types` +- `Table Operations` +- `Entity Operations` +- `Query Entities` + +## Non-hero scenarios + +- `Batch Operations`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#batch-operations`](non-hero-scenarios.md#batch-operations) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Data Types`: | Python Type | Table Storage Type | + See: [`non-hero-scenarios.md#data-types`](non-hero-scenarios.md#data-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..00a4e48b --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-data-tables-py/references/non-hero-scenarios.md @@ -0,0 +1,62 @@ +# azure-data-tables-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Batch Operations + +```python +from azure.data.tables import TableTransactionError + +# Batch operations (same partition only!) +operations = [ + ("create", {"PartitionKey": "batch", "RowKey": "1", "data": "first"}), + ("create", {"PartitionKey": "batch", "RowKey": "2", "data": "second"}), + ("upsert", {"PartitionKey": "batch", "RowKey": "3", "data": "third"}), +] + +try: + table_client.submit_transaction(operations) +except TableTransactionError as e: + print(f"Transaction failed: {e}") +``` + +## Async Client + +```python +from azure.data.tables.aio import TableServiceClient, TableClient +from azure.identity.aio import DefaultAzureCredential + +async def table_operations(): + async with DefaultAzureCredential() as credential: + async with TableClient( + endpoint="https://.table.core.windows.net", + table_name="mytable", + credential=credential + ) as client: + # Create + await client.create_entity(entity={ + "PartitionKey": "async", + "RowKey": "1", + "data": "test" + }) + + # Query + async for entity in client.query_entities("PartitionKey eq 'async'"): + print(entity) + +import asyncio +asyncio.run(table_operations()) +``` + +## Data Types + +| Python Type | Table Storage Type | +|-------------|-------------------| +| `str` | String | +| `int` | Int64 | +| `float` | Double | +| `bool` | Boolean | +| `datetime` | DateTime | +| `bytes` | Binary | +| `UUID` | Guid | diff --git a/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/SKILL.md index 907ee84b..bf34e1da 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/SKILL.md @@ -193,3 +193,10 @@ with EventGridPublisherClient( 7. **Use async client** for high-throughput scenarios 8. **Handle retries** — Event Grid has built-in retry 9. **Set appropriate event types** for routing and filtering + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/capabilities.md new file mode 100644 index 00000000..5c18e7d9 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/capabilities.md @@ -0,0 +1,32 @@ +# azure-eventgrid-py capability coverage + +**SDK/package**: `azure-eventgrid` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Event Types` +- `Publish CloudEvents` +- `Publish EventGridEvents` +- `Event Properties` + +## Non-hero scenarios + +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Namespace Topics (Event Grid Namespaces)`: For Event Grid Namespaces (pull delivery): + See: [`non-hero-scenarios.md#namespace-topics-event-grid-namespaces`](non-hero-scenarios.md#namespace-topics-event-grid-namespaces) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..a74c845f --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-eventgrid-py/references/non-hero-scenarios.md @@ -0,0 +1,47 @@ +# azure-eventgrid-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Async Client + +```python +from azure.eventgrid.aio import EventGridPublisherClient +from azure.identity.aio import DefaultAzureCredential + +async def publish_events(): + credential = DefaultAzureCredential() + + async with EventGridPublisherClient(endpoint, credential) as client: + event = CloudEvent( + type="MyApp.Events.Test", + source="/myapp", + data={"message": "hello"} + ) + await client.send(event) + +import asyncio +asyncio.run(publish_events()) +``` + +## Namespace Topics (Event Grid Namespaces) + +For Event Grid Namespaces (pull delivery): + +```python +from azure.eventgrid import EventGridPublisherClient +from azure.identity import DefaultAzureCredential + +# Namespace endpoint (different from custom topic) +namespace_endpoint = "https://..eventgrid.azure.net" +topic_name = "my-topic" + +with EventGridPublisherClient( + endpoint=namespace_endpoint, + credential=DefaultAzureCredential() +) as client: + client.send( + event, + namespace_topic=topic_name + ) +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-identity-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-identity-py/SKILL.md index 37e6a034..f187f844 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-identity-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-identity-py/SKILL.md @@ -62,7 +62,20 @@ AZURE_CLIENT_ID= AZURE_TOKEN_CREDENTIALS=dev|prod| # Optional, restricts DAC chain ``` -## DefaultAzureCredential +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **Prefer `DefaultAzureCredential`.** It works locally (Azure CLI / VS Code / Developer CLI) and in Azure (managed identity, workload identity) with no code change. Avoid connection strings, account/API keys — they bypass Entra audit and rotation. +> - Local dev: `DefaultAzureCredential` works as-is. +> - Production: set `AZURE_TOKEN_CREDENTIALS=prod` (or `AZURE_TOKEN_CREDENTIALS=`) to constrain the credential chain to production-safe credentials. +> 2. **Wrap credentials and clients in context managers** when they own token caches / transports: +> - Sync: `with DefaultAzureCredential() as credential:` +> - Async: `async with DefaultAzureCredential() as credential:` (from `azure.identity.aio`) +> +> Snippets may abbreviate this setup, but production code should always follow both rules. + +### DefaultAzureCredential The recommended credential for most scenarios. Tries multiple authentication methods in order: @@ -522,3 +535,10 @@ AZURE_LOG_LEVEL=debug | API Reference | https://learn.microsoft.com/python/api/azure-identity | | GitHub Source | https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity | | Credential Chains | https://aka.ms/azsdk/python/identity/credential-chains | + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/capabilities.md new file mode 100644 index 00000000..bfc31d6e --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/capabilities.md @@ -0,0 +1,42 @@ +# azure-identity-py capability coverage + +**SDK/package**: `azure-identity` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `get_bearer_token_provider` +- `Credential Types` +- `Specific Credential Examples` +- `Getting Tokens Directly` + +## Non-hero scenarios + +- `Async Credentials`: Async credentials are in `azure.identity.aio`. Always close them or use `async with`: + See: [`non-hero-scenarios.md#async-credentials`](non-hero-scenarios.md#async-credentials) +- `Sovereign Clouds`: Use `AzureAuthorityHosts` or the `AZURE_AUTHORITY_HOST` env var: + See: [`non-hero-scenarios.md#sovereign-clouds`](non-hero-scenarios.md#sovereign-clouds) +- `Persistent Token Caching`: Opt-in disk-based caching with `TokenCachePersistenceOptions`: + See: [`non-hero-scenarios.md#persistent-token-caching`](non-hero-scenarios.md#persistent-token-caching) +- `Multi-Tenant Support`: Allow token acquisition for additional tenants beyond the configured one: + See: [`non-hero-scenarios.md#multi-tenant-support`](non-hero-scenarios.md#multi-tenant-support) +- `Error Handling`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#error-handling`](non-hero-scenarios.md#error-handling) +- `Logging`: Enable authentication logging for debugging: + See: [`non-hero-scenarios.md#logging`](non-hero-scenarios.md#logging) +- `Credential Selection Matrix`: | Environment | Recommended Credential | + See: [`non-hero-scenarios.md#credential-selection-matrix`](non-hero-scenarios.md#credential-selection-matrix) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..e1c9ec06 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-identity-py/references/non-hero-scenarios.md @@ -0,0 +1,129 @@ +# azure-identity-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Async Credentials + +Async credentials are in `azure.identity.aio`. Always close them or use `async with`: + +```python +from azure.identity.aio import DefaultAzureCredential +from azure.storage.blob.aio import BlobServiceClient + +async def main(): + # Preferred: use async context manager for both credential and client + async with DefaultAzureCredential() as credential: + async with BlobServiceClient( + account_url="https://.blob.core.windows.net", + credential=credential, + ) as client: + # ... async operations + pass +``` + +> The async `get_bearer_token_provider` is at `azure.identity.aio.get_bearer_token_provider`. + +## Sovereign Clouds + +Use `AzureAuthorityHosts` or the `AZURE_AUTHORITY_HOST` env var: + +```python +from azure.identity import DefaultAzureCredential, AzureAuthorityHosts + +# Azure Government +credential = DefaultAzureCredential(authority=AzureAuthorityHosts.AZURE_GOVERNMENT) + +# Azure China +credential = DefaultAzureCredential(authority=AzureAuthorityHosts.AZURE_CHINA) +``` + +| Constant | Authority | +|----------|-----------| +| `AzureAuthorityHosts.AZURE_PUBLIC_CLOUD` | `login.microsoftonline.com` (default) | +| `AzureAuthorityHosts.AZURE_GOVERNMENT` | `login.microsoftonline.us` | +| `AzureAuthorityHosts.AZURE_CHINA` | `login.chinacloudapi.cn` | + +## Persistent Token Caching + +Opt-in disk-based caching with `TokenCachePersistenceOptions`: + +```python +from azure.identity import DefaultAzureCredential, TokenCachePersistenceOptions + +credential = DefaultAzureCredential( + cache_persistence_options=TokenCachePersistenceOptions() +) + +# Allow unencrypted fallback (NOT recommended for production) +credential = DefaultAzureCredential( + cache_persistence_options=TokenCachePersistenceOptions(allow_unencrypted_storage=True) +) +``` + +Storage: Windows (DPAPI), macOS (Keychain), Linux (Keyring). + +## Multi-Tenant Support + +Allow token acquisition for additional tenants beyond the configured one: + +```python +from azure.identity import ClientSecretCredential + +credential = ClientSecretCredential( + tenant_id="", + client_id="", + client_secret="", + additionally_allowed_tenants=["", "*"], # "*" allows any tenant +) +``` + +## Error Handling + +```python +from azure.identity import DefaultAzureCredential, CredentialUnavailableError +from azure.core.exceptions import ClientAuthenticationError + +with DefaultAzureCredential() as credential: + try: + token = credential.get_token("https://management.azure.com/.default") + except CredentialUnavailableError: + # No credential in the chain could attempt authentication + pass + except ClientAuthenticationError as e: + # Authentication was attempted but failed + # e.message contains details from each credential in the chain + pass +``` + +## Logging + +Enable authentication logging for debugging: + +```python +import logging + +# Enable verbose Azure Identity logging +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger("azure.identity") +logger.setLevel(logging.DEBUG) +``` + +```bash +# Or via environment variable +AZURE_LOG_LEVEL=debug +``` + +## Credential Selection Matrix + +| Environment | Recommended Credential | +|-------------|------------------------| +| Local Development | `DefaultAzureCredential` (uses Azure CLI) | +| Azure App Service | `DefaultAzureCredential` (uses Managed Identity) | +| Azure Functions | `DefaultAzureCredential` (uses Managed Identity) | +| Azure Kubernetes Service | `WorkloadIdentityCredential` | +| Azure VMs | `DefaultAzureCredential` (uses Managed Identity) | +| CI/CD Pipeline | `EnvironmentCredential` or `AzurePipelinesCredential` | +| Desktop App | `InteractiveBrowserCredential` | +| CLI / Headless Tool | `DeviceCodeCredential` | +| Middle-tier Service | `OnBehalfOfCredential` | diff --git a/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/SKILL.md index 20c836c7..ef10d433 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/SKILL.md @@ -272,3 +272,10 @@ except HttpResponseError as e: 8. **Use Key Vault references** in App Service/Functions config 9. **Cache secrets** appropriately to reduce API calls 10. **Use async clients** for high-throughput scenarios + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/capabilities.md new file mode 100644 index 00000000..5a008b97 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/capabilities.md @@ -0,0 +1,32 @@ +# azure-keyvault-py capability coverage + +**SDK/package**: `azure-keyvault-secrets, azure-keyvault-keys, azure-keyvault-certificates` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Secrets` +- `Keys` +- `Certificates` +- `Client Types Table` + +## Non-hero scenarios + +- `Async Clients`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-clients`](non-hero-scenarios.md#async-clients) +- `Error Handling`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#error-handling`](non-hero-scenarios.md#error-handling) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..2f1bd32f --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-keyvault-py/references/non-hero-scenarios.md @@ -0,0 +1,35 @@ +# azure-keyvault-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Async Clients + +```python +from azure.identity.aio import DefaultAzureCredential +from azure.keyvault.secrets.aio import SecretClient + +async def get_secret(): + async with DefaultAzureCredential() as credential: + async with SecretClient(vault_url=vault_url, credential=credential) as client: + secret = await client.get_secret("my-secret") + print(secret.value) + +import asyncio +asyncio.run(get_secret()) +``` + +## Error Handling + +```python +from azure.core.exceptions import ResourceNotFoundError, HttpResponseError + +try: + secret = client.get_secret("nonexistent") +except ResourceNotFoundError: + print("Secret not found") +except HttpResponseError as e: + if e.status_code == 403: + print("Access denied - check RBAC permissions") + raise +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/SKILL.md index 8cdd7c32..20cf2ada 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/SKILL.md @@ -255,3 +255,10 @@ async def broadcast(): 7. **Handle reconnection** in client applications 8. **Use JSON** content type for structured data 9. **Close connections** gracefully with reasons + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/capabilities.md new file mode 100644 index 00000000..6964e572 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/capabilities.md @@ -0,0 +1,30 @@ +# azure-messaging-webpubsubservice-py capability coverage + +**SDK/package**: `azure-messaging-webpubsubservice` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Service Client (Server-Side)` +- `Client SDK (Python WebSocket Client)` +- `Async Service Client` +- `Client Operations` + +## Non-hero scenarios + +- `Operational hardening`: Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. + See: [`non-hero-scenarios.md#operational-hardening`](non-hero-scenarios.md#operational-hardening) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..56754214 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py/references/non-hero-scenarios.md @@ -0,0 +1,8 @@ +# azure-messaging-webpubsubservice-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Operational hardening + +Use this section for retries, timeouts, pagination, and cleanup patterns specific to this SDK. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/SKILL.md index 99f050f5..97f22bbd 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/SKILL.md @@ -89,7 +89,7 @@ for api_center in api_centers: ## Register an API ```python -from azure.mgmt.apicenter.models import Api, ApiKind, LifecycleStage +from azure.mgmt.apicenter.models import Api, ApiKind, ApiProperties api = client.apis.create_or_update( resource_group_name="my-resource-group", @@ -97,22 +97,23 @@ api = client.apis.create_or_update( workspace_name="default", api_name="my-api", resource=Api( - title="My API", - description="A sample API for demonstration", - kind=ApiKind.REST, - lifecycle_stage=LifecycleStage.PRODUCTION, - terms_of_service={"url": "https://example.com/terms"}, - contacts=[{"name": "API Team", "email": "api-team@example.com"}] - ) + properties=ApiProperties( + title="My API", + description="A sample API for demonstration", + kind=ApiKind.REST, + terms_of_service={"url": "https://example.com/terms"}, + contacts=[{"name": "API Team", "email": "api-team@example.com"}], + ) + ), ) -print(f"Registered API: {api.title}") +print(f"Registered API: {api.properties.title}") ``` ## Create API Version ```python -from azure.mgmt.apicenter.models import ApiVersion, LifecycleStage +from azure.mgmt.apicenter.models import ApiVersion, ApiVersionProperties, LifecycleStage version = client.api_versions.create_or_update( resource_group_name="my-resource-group", @@ -121,18 +122,20 @@ version = client.api_versions.create_or_update( api_name="my-api", version_name="v1", resource=ApiVersion( - title="Version 1.0", - lifecycle_stage=LifecycleStage.PRODUCTION - ) + properties=ApiVersionProperties( + title="Version 1.0", + lifecycle_stage=LifecycleStage.PRODUCTION, + ) + ), ) -print(f"Created version: {version.title}") +print(f"Created version: {version.properties.title}") ``` ## Add API Definition ```python -from azure.mgmt.apicenter.models import ApiDefinition +from azure.mgmt.apicenter.models import ApiDefinition, ApiDefinitionProperties definition = client.api_definitions.create_or_update( resource_group_name="my-resource-group", @@ -142,9 +145,11 @@ definition = client.api_definitions.create_or_update( version_name="v1", definition_name="openapi", resource=ApiDefinition( - title="OpenAPI Definition", - description="OpenAPI 3.0 specification" - ) + properties=ApiDefinitionProperties( + title="OpenAPI Definition", + description="OpenAPI 3.0 specification", + ) + ), ) ``` @@ -154,7 +159,7 @@ definition = client.api_definitions.create_or_update( from azure.mgmt.apicenter.models import ApiSpecImportRequest, ApiSpecImportSourceFormat # Import from inline content -client.api_definitions.import_specification( +client.api_definitions.begin_import_specification( resource_group_name="my-resource-group", service_name="my-api-center", workspace_name="default", @@ -163,9 +168,9 @@ client.api_definitions.import_specification( definition_name="openapi", body=ApiSpecImportRequest( format=ApiSpecImportSourceFormat.INLINE, - value='{"openapi": "3.0.0", "info": {"title": "My API", "version": "1.0"}, "paths": {}}' + value='{"openapi": "3.0.0", "info": {"title": "My API", "version": "1.0"}, "paths": {}}', ) -) +).result() ``` ## List APIs @@ -184,7 +189,7 @@ for api in apis: ## Create Environment ```python -from azure.mgmt.apicenter.models import Environment, EnvironmentKind +from azure.mgmt.apicenter.models import Environment, EnvironmentKind, EnvironmentProperties environment = client.environments.create_or_update( resource_group_name="my-resource-group", @@ -192,18 +197,20 @@ environment = client.environments.create_or_update( workspace_name="default", environment_name="production", resource=Environment( - title="Production", - description="Production environment", - kind=EnvironmentKind.PRODUCTION, - server={"type": "Azure API Management", "management_portal_uri": ["https://portal.azure.com"]} - ) + properties=EnvironmentProperties( + title="Production", + description="Production environment", + kind=EnvironmentKind.PRODUCTION, + server={"type": "Azure API Management", "management_portal_uri": ["https://portal.azure.com"]}, + ) + ), ) ``` ## Create Deployment ```python -from azure.mgmt.apicenter.models import Deployment, DeploymentState +from azure.mgmt.apicenter.models import Deployment, DeploymentProperties, DeploymentState deployment = client.deployments.create_or_update( resource_group_name="my-resource-group", @@ -212,28 +219,32 @@ deployment = client.deployments.create_or_update( api_name="my-api", deployment_name="prod-deployment", resource=Deployment( - title="Production Deployment", - description="Deployed to production APIM", - environment_id="/workspaces/default/environments/production", - definition_id="/workspaces/default/apis/my-api/versions/v1/definitions/openapi", - state=DeploymentState.ACTIVE, - server={"runtime_uri": ["https://api.example.com"]} - ) + properties=DeploymentProperties( + title="Production Deployment", + description="Deployed to production APIM", + environment_id="/workspaces/default/environments/production", + definition_id="/workspaces/default/apis/my-api/versions/v1/definitions/openapi", + state=DeploymentState.ACTIVE, + server={"runtime_uri": ["https://api.example.com"]}, + ) + ), ) ``` ## Define Custom Metadata ```python -from azure.mgmt.apicenter.models import MetadataSchema +from azure.mgmt.apicenter.models import MetadataSchema, MetadataSchemaProperties metadata = client.metadata_schemas.create_or_update( resource_group_name="my-resource-group", service_name="my-api-center", metadata_schema_name="data-classification", resource=MetadataSchema( - schema='{"type": "string", "title": "Data Classification", "enum": ["public", "internal", "confidential"]}' - ) + properties=MetadataSchemaProperties( + schema='{"type": "string", "title": "Data Classification", "enum": ["public", "internal", "confidential"]}' + ) + ), ) ``` @@ -266,3 +277,10 @@ metadata = client.metadata_schemas.create_or_update( 6. **Import specifications** to enable API analysis and linting 7. **Use lifecycle stages** to track API maturity 8. **Add contacts** for API ownership and support + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/capabilities.md new file mode 100644 index 00000000..cca21654 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/capabilities.md @@ -0,0 +1,44 @@ +# azure-mgmt-apicenter-py capability coverage + +**SDK/package**: `azure-mgmt-apicenter` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Create API Center` +- `List API Centers` +- `Register an API` +- `Create API Version` + +## Non-hero scenarios + +- `Add API Definition`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#add-api-definition`](non-hero-scenarios.md#add-api-definition) +- `Import API Specification`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#import-api-specification`](non-hero-scenarios.md#import-api-specification) +- `List APIs`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-apis`](non-hero-scenarios.md#list-apis) +- `Create Environment`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-environment`](non-hero-scenarios.md#create-environment) +- `Create Deployment`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-deployment`](non-hero-scenarios.md#create-deployment) +- `Define Custom Metadata`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#define-custom-metadata`](non-hero-scenarios.md#define-custom-metadata) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) +- `Operations`: | Operation Group | Purpose | + See: [`non-hero-scenarios.md#operations`](non-hero-scenarios.md#operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..1fb05fd8 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py/references/non-hero-scenarios.md @@ -0,0 +1,139 @@ +# azure-mgmt-apicenter-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Add API Definition + +```python +from azure.mgmt.apicenter.models import ApiDefinition, ApiDefinitionProperties + +definition = client.api_definitions.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1", + definition_name="openapi", + resource=ApiDefinition( + properties=ApiDefinitionProperties( + title="OpenAPI Definition", + description="OpenAPI 3.0 specification", + ) + ), +) +``` + +## Import API Specification + +```python +from azure.mgmt.apicenter.models import ApiSpecImportRequest, ApiSpecImportSourceFormat + +# Import from inline content +client.api_definitions.begin_import_specification( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1", + definition_name="openapi", + body=ApiSpecImportRequest( + format=ApiSpecImportSourceFormat.INLINE, + value='{"openapi": "3.0.0", "info": {"title": "My API", "version": "1.0"}, "paths": {}}', + ) +).result() +``` + +## List APIs + +```python +apis = client.apis.list( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default" +) + +for api in apis: + print(f"{api.name}: {api.title} ({api.kind})") +``` + +## Create Environment + +```python +from azure.mgmt.apicenter.models import Environment, EnvironmentKind, EnvironmentProperties + +environment = client.environments.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + environment_name="production", + resource=Environment( + properties=EnvironmentProperties( + title="Production", + description="Production environment", + kind=EnvironmentKind.PRODUCTION, + server={"type": "Azure API Management", "management_portal_uri": ["https://portal.azure.com"]}, + ) + ), +) +``` + +## Create Deployment + +```python +from azure.mgmt.apicenter.models import Deployment, DeploymentProperties, DeploymentState + +deployment = client.deployments.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + deployment_name="prod-deployment", + resource=Deployment( + properties=DeploymentProperties( + title="Production Deployment", + description="Deployed to production APIM", + environment_id="/workspaces/default/environments/production", + definition_id="/workspaces/default/apis/my-api/versions/v1/definitions/openapi", + state=DeploymentState.ACTIVE, + server={"runtime_uri": ["https://api.example.com"]}, + ) + ), +) +``` + +## Define Custom Metadata + +```python +from azure.mgmt.apicenter.models import MetadataSchema, MetadataSchemaProperties + +metadata = client.metadata_schemas.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + metadata_schema_name="data-classification", + resource=MetadataSchema( + properties=MetadataSchemaProperties( + schema='{"type": "string", "title": "Data Classification", "enum": ["public", "internal", "confidential"]}' + ) + ), +) +``` + +## Client Types + +| Client | Purpose | +|--------|---------| +| `ApiCenterMgmtClient` | Main client for all operations | + +## Operations + +| Operation Group | Purpose | +|----------------|---------| +| `services` | API Center service management | +| `workspaces` | Workspace management | +| `apis` | API registration and management | +| `api_versions` | API version management | +| `api_definitions` | API definition management | +| `deployments` | Deployment tracking | +| `environments` | Environment management | +| `metadata_schemas` | Custom metadata definitions | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/SKILL.md index 23cccf5a..ddec4896 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/SKILL.md @@ -302,3 +302,10 @@ user = client.user.create_or_update( 6. **Enable Application Insights** for monitoring 7. **Use backends** to abstract backend services 8. **Version your APIs** using APIM's versioning features + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/capabilities.md new file mode 100644 index 00000000..a74cb701 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/capabilities.md @@ -0,0 +1,44 @@ +# azure-mgmt-apimanagement-py capability coverage + +**SDK/package**: `azure-mgmt-apimanagement` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Create APIM Service` +- `Import API from OpenAPI` +- `Import API from URL` +- `List APIs` + +## Non-hero scenarios + +- `Create Product`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-product`](non-hero-scenarios.md#create-product) +- `Add API to Product`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#add-api-to-product`](non-hero-scenarios.md#add-api-to-product) +- `Create Subscription`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-subscription`](non-hero-scenarios.md#create-subscription) +- `Set API Policy`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#set-api-policy`](non-hero-scenarios.md#set-api-policy) +- `Create Named Value (Secret)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-named-value-secret`](non-hero-scenarios.md#create-named-value-secret) +- `Create Backend`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-backend`](non-hero-scenarios.md#create-backend) +- `Create User`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#create-user`](non-hero-scenarios.md#create-user) +- `Operation Groups`: | Group | Purpose | + See: [`non-hero-scenarios.md#operation-groups`](non-hero-scenarios.md#operation-groups) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..fa31a077 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py/references/non-hero-scenarios.md @@ -0,0 +1,156 @@ +# azure-mgmt-apimanagement-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Create Product + +```python +from azure.mgmt.apimanagement.models import ProductContract + +product = client.product.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + product_id="premium", + parameters=ProductContract( + display_name="Premium", + description="Premium tier with unlimited access", + subscription_required=True, + approval_required=False, + state="published" + ) +) + +print(f"Created product: {product.display_name}") +``` + +## Add API to Product + +```python +client.product_api.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + product_id="premium", + api_id="my-api" +) +``` + +## Create Subscription + +```python +from azure.mgmt.apimanagement.models import SubscriptionCreateParameters + +subscription = client.subscription.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + sid="my-subscription", + parameters=SubscriptionCreateParameters( + display_name="My Subscription", + scope=f"/products/premium", + state="active" + ) +) + +print(f"Subscription key: {subscription.primary_key}") +``` + +## Set API Policy + +```python +from azure.mgmt.apimanagement.models import PolicyContract + +policy_xml = """ + + + + + CustomValue + + + + + + + + +""" + +client.api_policy.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + api_id="my-api", + policy_id="policy", + parameters=PolicyContract( + value=policy_xml, + format="xml" + ) +) +``` + +## Create Named Value (Secret) + +```python +from azure.mgmt.apimanagement.models import NamedValueCreateContract + +named_value = client.named_value.begin_create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + named_value_id="backend-api-key", + parameters=NamedValueCreateContract( + display_name="Backend API Key", + value="secret-key-value", + secret=True + ) +).result() +``` + +## Create Backend + +```python +from azure.mgmt.apimanagement.models import BackendContract + +backend = client.backend.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + backend_id="my-backend", + parameters=BackendContract( + url="https://api.backend.example.com", + protocol="http", + description="My backend service" + ) +) +``` + +## Create User + +```python +from azure.mgmt.apimanagement.models import UserCreateParameters + +user = client.user.create_or_update( + resource_group_name="my-resource-group", + service_name="my-apim", + user_id="newuser", + parameters=UserCreateParameters( + email="user@example.com", + first_name="John", + last_name="Doe" + ) +) +``` + +## Operation Groups + +| Group | Purpose | +|-------|---------| +| `api_management_service` | APIM instance management | +| `api` | API operations | +| `api_operation` | API operation details | +| `api_policy` | API-level policies | +| `product` | Product management | +| `product_api` | Product-API associations | +| `subscription` | Subscription management | +| `user` | User management | +| `named_value` | Named values/secrets | +| `backend` | Backend services | +| `certificate` | Certificates | +| `gateway` | Self-hosted gateways | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/SKILL.md index ec8513af..4ad0b650 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/SKILL.md @@ -342,3 +342,10 @@ for conn in connections: 7. **Rotate Direct Line keys** periodically 8. **Use managed identity** when possible for bot connections 9. **Configure proper CORS** for Web Chat channel + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/capabilities.md new file mode 100644 index 00000000..9cdefab7 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/capabilities.md @@ -0,0 +1,46 @@ +# azure-mgmt-botservice-py capability coverage + +**SDK/package**: `azure-mgmt-botservice` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Create a Bot` +- `Get Bot Details` +- `List Bots in Resource Group` +- `List All Bots in Subscription` + +## Non-hero scenarios + +- `Update Bot`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#update-bot`](non-hero-scenarios.md#update-bot) +- `Delete Bot`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#delete-bot`](non-hero-scenarios.md#delete-bot) +- `Configure Channels`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#configure-channels`](non-hero-scenarios.md#configure-channels) +- `Get Channel Details`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#get-channel-details`](non-hero-scenarios.md#get-channel-details) +- `List Channel Keys`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-channel-keys`](non-hero-scenarios.md#list-channel-keys) +- `Bot Connections (OAuth)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#bot-connections-oauth`](non-hero-scenarios.md#bot-connections-oauth) +- `Client Operations`: | Operation | Method | + See: [`non-hero-scenarios.md#client-operations`](non-hero-scenarios.md#client-operations) +- `SKU Options`: | SKU | Description | + See: [`non-hero-scenarios.md#sku-options`](non-hero-scenarios.md#sku-options) +- `Channel Types`: | Channel | Class | Purpose | + See: [`non-hero-scenarios.md#channel-types`](non-hero-scenarios.md#channel-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..024f6a72 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py/references/non-hero-scenarios.md @@ -0,0 +1,208 @@ +# azure-mgmt-botservice-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Update Bot + +```python +bot = client.bots.update( + resource_group_name=resource_group, + resource_name=bot_name, + properties=BotProperties( + display_name="Updated Bot Name", + description="Updated description" + ) +) +``` + +## Delete Bot + +```python +client.bots.delete( + resource_group_name=resource_group, + resource_name=bot_name +) +``` + +## Configure Channels + +### Add Teams Channel + +```python +from azure.mgmt.botservice.models import ( + BotChannel, + MsTeamsChannel, + MsTeamsChannelProperties +) + +channel = client.channels.create( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="MsTeamsChannel", + parameters=BotChannel( + location="global", + properties=MsTeamsChannel( + properties=MsTeamsChannelProperties( + is_enabled=True + ) + ) + ) +) +``` + +### Add Direct Line Channel + +```python +from azure.mgmt.botservice.models import ( + BotChannel, + DirectLineChannel, + DirectLineChannelProperties, + DirectLineSite +) + +channel = client.channels.create( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="DirectLineChannel", + parameters=BotChannel( + location="global", + properties=DirectLineChannel( + properties=DirectLineChannelProperties( + sites=[ + DirectLineSite( + site_name="Default Site", + is_enabled=True, + is_v1_enabled=False, + is_v3_enabled=True + ) + ] + ) + ) + ) +) +``` + +### Add Web Chat Channel + +```python +from azure.mgmt.botservice.models import ( + BotChannel, + WebChatChannel, + WebChatChannelProperties, + WebChatSite +) + +channel = client.channels.create( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="WebChatChannel", + parameters=BotChannel( + location="global", + properties=WebChatChannel( + properties=WebChatChannelProperties( + sites=[ + WebChatSite( + site_name="Default Site", + is_enabled=True + ) + ] + ) + ) + ) +) +``` + +## Get Channel Details + +```python +channel = client.channels.get( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="DirectLineChannel" +) +``` + +## List Channel Keys + +```python +keys = client.channels.list_with_keys( + resource_group_name=resource_group, + resource_name=bot_name, + channel_name="DirectLineChannel" +) + +# Access Direct Line keys +if hasattr(keys.properties, 'properties'): + for site in keys.properties.properties.sites: + print(f"Site: {site.site_name}") + print(f"Key: {site.key}") +``` + +## Bot Connections (OAuth) + +### Create Connection Setting + +```python +from azure.mgmt.botservice.models import ( + ConnectionSetting, + ConnectionSettingProperties +) + +connection = client.bot_connection.create( + resource_group_name=resource_group, + resource_name=bot_name, + connection_name="graph-connection", + parameters=ConnectionSetting( + location="global", + properties=ConnectionSettingProperties( + client_id="", + client_secret="", + scopes="User.Read", + service_provider_id="" + ) + ) +) +``` + +### List Connections + +```python +connections = client.bot_connection.list_by_bot_service( + resource_group_name=resource_group, + resource_name=bot_name +) + +for conn in connections: + print(f"Connection: {conn.name}") +``` + +## Client Operations + +| Operation | Method | +|-----------|--------| +| `client.bots` | Bot CRUD operations | +| `client.channels` | Channel configuration | +| `client.bot_connection` | OAuth connection settings | +| `client.direct_line` | Direct Line channel operations | +| `client.email` | Email channel operations | +| `client.operations` | Available operations | +| `client.host_settings` | Host settings operations | + +## SKU Options + +| SKU | Description | +|-----|-------------| +| `F0` | Free tier (limited messages) | +| `S1` | Standard tier (unlimited messages) | + +## Channel Types + +| Channel | Class | Purpose | +|---------|-------|---------| +| `MsTeamsChannel` | Microsoft Teams | Teams integration | +| `DirectLineChannel` | Direct Line | Custom client integration | +| `WebChatChannel` | Web Chat | Embeddable web widget | +| `SlackChannel` | Slack | Slack workspace integration | +| `FacebookChannel` | Facebook | Messenger integration | +| `EmailChannel` | Email | Email communication | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/SKILL.md index fbb6c37b..31cb7e05 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/SKILL.md @@ -280,3 +280,10 @@ capacity = poller.result() 8. **Handle LRO properly** — don't assume immediate completion 9. **Set up capacity admins** — specify users who can manage workspaces 10. **Monitor capacity usage** via Azure Monitor metrics + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/capabilities.md new file mode 100644 index 00000000..2671e9f6 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/capabilities.md @@ -0,0 +1,48 @@ +# azure-mgmt-fabric-py capability coverage + +**SDK/package**: `azure-mgmt-fabric` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Create Fabric Capacity` +- `Get Capacity Details` +- `List Capacities in Resource Group` +- `List All Capacities in Subscription` + +## Non-hero scenarios + +- `Update Capacity`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#update-capacity`](non-hero-scenarios.md#update-capacity) +- `Suspend Capacity`: Pause capacity to stop billing: + See: [`non-hero-scenarios.md#suspend-capacity`](non-hero-scenarios.md#suspend-capacity) +- `Resume Capacity`: Resume a paused capacity: + See: [`non-hero-scenarios.md#resume-capacity`](non-hero-scenarios.md#resume-capacity) +- `Delete Capacity`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#delete-capacity`](non-hero-scenarios.md#delete-capacity) +- `Check Name Availability`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#check-name-availability`](non-hero-scenarios.md#check-name-availability) +- `List Available SKUs`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-available-skus`](non-hero-scenarios.md#list-available-skus) +- `Client Operations`: | Operation | Method | + See: [`non-hero-scenarios.md#client-operations`](non-hero-scenarios.md#client-operations) +- `Fabric SKUs`: | SKU | Description | CUs | + See: [`non-hero-scenarios.md#fabric-skus`](non-hero-scenarios.md#fabric-skus) +- `Capacity States`: | State | Description | + See: [`non-hero-scenarios.md#capacity-states`](non-hero-scenarios.md#capacity-states) +- `Long-Running Operations`: All mutating operations are long-running (LRO). Use `.result()` to wait: + See: [`non-hero-scenarios.md#long-running-operations`](non-hero-scenarios.md#long-running-operations) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..cffbf63d --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py/references/non-hero-scenarios.md @@ -0,0 +1,142 @@ +# azure-mgmt-fabric-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Update Capacity + +```python +from azure.mgmt.fabric.models import FabricCapacityUpdate, CapacitySku + +updated = client.fabric_capacities.begin_update( + resource_group_name=resource_group, + capacity_name=capacity_name, + properties=FabricCapacityUpdate( + sku=CapacitySku( + name="F4", # Scale up + tier="Fabric" + ), + tags={"environment": "production"} + ) +).result() + +print(f"Updated SKU: {updated.sku.name}") +``` + +## Suspend Capacity + +Pause capacity to stop billing: + +```python +client.fabric_capacities.begin_suspend( + resource_group_name=resource_group, + capacity_name=capacity_name +).result() + +print("Capacity suspended") +``` + +## Resume Capacity + +Resume a paused capacity: + +```python +client.fabric_capacities.begin_resume( + resource_group_name=resource_group, + capacity_name=capacity_name +).result() + +print("Capacity resumed") +``` + +## Delete Capacity + +```python +client.fabric_capacities.begin_delete( + resource_group_name=resource_group, + capacity_name=capacity_name +).result() + +print("Capacity deleted") +``` + +## Check Name Availability + +```python +from azure.mgmt.fabric.models import CheckNameAvailabilityRequest + +result = client.fabric_capacities.check_name_availability( + location="eastus", + body=CheckNameAvailabilityRequest( + name="my-new-capacity", + type="Microsoft.Fabric/capacities" + ) +) + +if result.name_available: + print("Name is available") +else: + print(f"Name not available: {result.reason}") +``` + +## List Available SKUs + +```python +skus = client.fabric_capacities.list_skus( + resource_group_name=resource_group, + capacity_name=capacity_name +) + +for sku in skus: + print(f"SKU: {sku.name} - Tier: {sku.tier}") +``` + +## Client Operations + +| Operation | Method | +|-----------|--------| +| `client.fabric_capacities` | Capacity CRUD operations | +| `client.operations` | List available operations | + +## Fabric SKUs + +| SKU | Description | CUs | +|-----|-------------|-----| +| `F2` | Entry level | 2 Capacity Units | +| `F4` | Small | 4 Capacity Units | +| `F8` | Medium | 8 Capacity Units | +| `F16` | Large | 16 Capacity Units | +| `F32` | X-Large | 32 Capacity Units | +| `F64` | 2X-Large | 64 Capacity Units | +| `F128` | 4X-Large | 128 Capacity Units | +| `F256` | 8X-Large | 256 Capacity Units | +| `F512` | 16X-Large | 512 Capacity Units | +| `F1024` | 32X-Large | 1024 Capacity Units | +| `F2048` | 64X-Large | 2048 Capacity Units | + +## Capacity States + +| State | Description | +|-------|-------------| +| `Active` | Capacity is running | +| `Paused` | Capacity is suspended (no billing) | +| `Provisioning` | Being created | +| `Updating` | Being modified | +| `Deleting` | Being removed | +| `Failed` | Operation failed | + +## Long-Running Operations + +All mutating operations are long-running (LRO). Use `.result()` to wait: + +```python +# Synchronous wait +capacity = client.fabric_capacities.begin_create_or_update(...).result() + +# Or poll manually +poller = client.fabric_capacities.begin_create_or_update(...) +while not poller.done(): + print(f"Status: {poller.status()}") + time.sleep(5) +capacity = poller.result() +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/SKILL.md index b6146061..56f0f86d 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/SKILL.md @@ -228,3 +228,10 @@ Stream names follow patterns: 7. **Use async client** for high-throughput scenarios 8. **Batch uploads** — SDK handles batching, but send reasonable chunks 9. **Monitor ingestion** — Check Log Analytics for ingestion status + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/capabilities.md new file mode 100644 index 00000000..be938859 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/capabilities.md @@ -0,0 +1,40 @@ +# azure-monitor-ingestion-py capability coverage + +**SDK/package**: `azure-monitor-ingestion` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Upload Custom Logs` +- `Upload from JSON File` +- `Custom Error Handling` +- `Ignore Errors` + +## Non-hero scenarios + +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Sovereign Clouds`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#sovereign-clouds`](non-hero-scenarios.md#sovereign-clouds) +- `Batching Behavior`: The SDK automatically: + See: [`non-hero-scenarios.md#batching-behavior`](non-hero-scenarios.md#batching-behavior) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) +- `Key Concepts`: | Concept | Description | + See: [`non-hero-scenarios.md#key-concepts`](non-hero-scenarios.md#key-concepts) +- `DCR Stream Name Format`: Stream names follow patterns: + See: [`non-hero-scenarios.md#dcr-stream-name-format`](non-hero-scenarios.md#dcr-stream-name-format) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..620e311e --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py/references/non-hero-scenarios.md @@ -0,0 +1,73 @@ +# azure-monitor-ingestion-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Async Client + +```python +import asyncio +from azure.monitor.ingestion.aio import LogsIngestionClient +from azure.identity.aio import DefaultAzureCredential + +async def upload_logs(): + async with LogsIngestionClient( + endpoint=endpoint, + credential=DefaultAzureCredential() + ) as client: + await client.upload( + rule_id=rule_id, + stream_name=stream_name, + logs=logs + ) + +asyncio.run(upload_logs()) +``` + +## Sovereign Clouds + +```python +from azure.identity import AzureAuthorityHosts, DefaultAzureCredential +from azure.monitor.ingestion import LogsIngestionClient + +# Azure Government +credential = DefaultAzureCredential(authority=AzureAuthorityHosts.AZURE_GOVERNMENT) +with LogsIngestionClient( + endpoint="https://example.ingest.monitor.azure.us", + credential=credential, + credential_scopes=["https://monitor.azure.us/.default"] +) as client: + # client.upload(...) + ... +``` + +## Batching Behavior + +The SDK automatically: +- Splits logs into chunks of 1MB or less +- Compresses each chunk with gzip +- Uploads chunks in parallel + +No manual batching needed for large log sets. + +## Client Types + +| Client | Purpose | +|--------|---------| +| `LogsIngestionClient` | Sync client for uploading logs | +| `LogsIngestionClient` (aio) | Async client for uploading logs | + +## Key Concepts + +| Concept | Description | +|---------|-------------| +| **DCE** | Data Collection Endpoint — ingestion URL | +| **DCR** | Data Collection Rule — defines schema, transformations, destination | +| **Stream** | Named data flow within a DCR | +| **Custom Table** | Target table in Log Analytics (ends with `_CL`) | + +## DCR Stream Name Format + +Stream names follow patterns: +- `Custom-_CL` — For custom tables +- `Microsoft-` — For built-in tables diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/SKILL.md index bd9553c4..86f904ba 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/SKILL.md @@ -27,7 +27,16 @@ APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=xxx;IngestionEndpoint=h AZURE_TOKEN_CREDENTIALS=prod # Required only if DefaultAzureCredential is used in production ``` -> **🔑 Auth & lifecycle:** These exporters take a connection string by design, but for *AAD-authenticated ingestion* (where supported) prefer `DefaultAzureCredential` via the `credential=` parameter — see the [Azure AD Authentication](#azure-ad-authentication) section. Any Azure SDK clients you create alongside the exporter should be wrapped in `with`/`async with` blocks (and async credentials from `azure.identity.aio` likewise). +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **Prefer `DefaultAzureCredential` for ingestion auth when supported.** `APPLICATIONINSIGHTS_CONNECTION_STRING` identifies the target Application Insights resource, and `credential=DefaultAzureCredential(...)` provides Microsoft Entra authentication. +> - Local dev: `DefaultAzureCredential` works as-is. +> - Production: set `AZURE_TOKEN_CREDENTIALS=prod` (or `AZURE_TOKEN_CREDENTIALS=`) to constrain the credential chain to production-safe credentials. +> 2. **Providers are not context managers.** Flush and shut down telemetry providers explicitly at process exit so buffers are exported deterministically. +> +> Snippets may abbreviate this setup, but production code should always follow both rules. ## When to Use @@ -221,10 +230,17 @@ exporter = AzureMonitorTraceExporter( ## Best Practices 1. **Pick sync OR async and stay consistent.** Do not mix `azure.xxx` sync clients with `azure.xxx.aio` async clients in the same call path. Choose one mode per module. -2. **Flush and shut down providers at process exit.** Call the shutdown/flush APIs (e.g. `tracer_provider.shutdown()`, `meter_provider.shutdown()`, `logger_provider.shutdown()`) at process exit to flush telemetry before the process terminates. +2. **Call `provider.shutdown()` / `flush()` at process exit to flush telemetry — providers are not context managers.** 3. **Use BatchSpanProcessor** for production (not SimpleSpanProcessor) 4. **Use ApplicationInsightsSampler** for consistent sampling across services 5. **Enable offline storage** for reliability in production 6. **Use Microsoft Entra authentication** instead of instrumentation keys 7. **Set export intervals** appropriate for your workload 8. **Use the distro** (`azure-monitor-opentelemetry`) unless you need custom pipelines + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/capabilities.md new file mode 100644 index 00000000..6cddf922 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/capabilities.md @@ -0,0 +1,42 @@ +# azure-monitor-opentelemetry-exporter-py capability coverage + +**SDK/package**: `azure-monitor-opentelemetry-exporter` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Trace Exporter` +- `Metric Exporter` +- `Log Exporter` +- `From Environment Variable` + +## Non-hero scenarios + +- `Azure AD Authentication`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#azure-ad-authentication`](non-hero-scenarios.md#azure-ad-authentication) +- `Sampling`: Use `ApplicationInsightsSampler` for consistent sampling: + See: [`non-hero-scenarios.md#sampling`](non-hero-scenarios.md#sampling) +- `Offline Storage`: Configure offline storage for retry: + See: [`non-hero-scenarios.md#offline-storage`](non-hero-scenarios.md#offline-storage) +- `Disable Offline Storage`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#disable-offline-storage`](non-hero-scenarios.md#disable-offline-storage) +- `Sovereign Clouds`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#sovereign-clouds`](non-hero-scenarios.md#sovereign-clouds) +- `Exporter Types`: | Exporter | Telemetry Type | Application Insights Table | + See: [`non-hero-scenarios.md#exporter-types`](non-hero-scenarios.md#exporter-types) +- `Configuration Options`: | Parameter | Description | Default | + See: [`non-hero-scenarios.md#configuration-options`](non-hero-scenarios.md#configuration-options) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..566af556 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py/references/non-hero-scenarios.md @@ -0,0 +1,91 @@ +# azure-monitor-opentelemetry-exporter-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Azure AD Authentication + +```python +from azure.identity import DefaultAzureCredential, ManagedIdentityCredential +from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter + +# Local dev: DefaultAzureCredential. Production: set AZURE_TOKEN_CREDENTIALS=prod or AZURE_TOKEN_CREDENTIALS= +credential = DefaultAzureCredential(require_envvar=True) +# Or use a specific credential directly in production: +# See https://learn.microsoft.com/python/api/overview/azure/identity-readme?view=azure-python#credential-classes +# credential = ManagedIdentityCredential() + +exporter = AzureMonitorTraceExporter( + credential=credential +) +``` + +## Sampling + +Use `ApplicationInsightsSampler` for consistent sampling: + +```python +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatio +from azure.monitor.opentelemetry.exporter import ApplicationInsightsSampler + +# Sample 10% of traces +sampler = ApplicationInsightsSampler(sampling_ratio=0.1) + +trace.set_tracer_provider(TracerProvider(sampler=sampler)) +``` + +## Offline Storage + +Configure offline storage for retry: + +```python +from azure.identity import DefaultAzureCredential +from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter + +exporter = AzureMonitorTraceExporter( + credential=DefaultAzureCredential(), + storage_directory="/path/to/storage", # Custom storage path + disable_offline_storage=False # Enable retry (default) +) +``` + +## Disable Offline Storage + +```python +exporter = AzureMonitorTraceExporter( + credential=DefaultAzureCredential(), + disable_offline_storage=True # No retry on failure +) +``` + +## Sovereign Clouds + +```python +from azure.identity import AzureAuthorityHosts, DefaultAzureCredential +from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter + +# Azure Government +credential = DefaultAzureCredential(authority=AzureAuthorityHosts.AZURE_GOVERNMENT) +exporter = AzureMonitorTraceExporter( + connection_string="InstrumentationKey=xxx;IngestionEndpoint=https://xxx.in.applicationinsights.azure.us/", + credential=credential +) +``` + +## Exporter Types + +| Exporter | Telemetry Type | Application Insights Table | +|----------|---------------|---------------------------| +| `AzureMonitorTraceExporter` | Traces/Spans | requests, dependencies, exceptions | +| `AzureMonitorMetricExporter` | Metrics | customMetrics, performanceCounters | +| `AzureMonitorLogExporter` | Logs | traces, customEvents | + +## Configuration Options + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `connection_string` | Application Insights connection string | From env var | +| `credential` | Azure credential for AAD auth | None | +| `disable_offline_storage` | Disable retry storage | False | +| `storage_directory` | Custom storage path | Temp directory | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/SKILL.md index 00f1d753..ef0b2e0f 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/SKILL.md @@ -27,7 +27,16 @@ APPLICATIONINSIGHTS_CONNECTION_STRING=InstrumentationKey=xxx;IngestionEndpoint=h AZURE_TOKEN_CREDENTIALS=prod # Required only if DefaultAzureCredential is used in production ``` -> **🔑 Auth & lifecycle:** This distro is configured with a connection string by design, but for *AAD-authenticated ingestion* (where supported) prefer `DefaultAzureCredential` via the `credential=` parameter — see the [Azure AD Authentication](#azure-ad-authentication) section. Any Azure SDK clients you create alongside the exporter should be wrapped in `with`/`async with` blocks (and async credentials from `azure.identity.aio` likewise). +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **Prefer `DefaultAzureCredential` for ingestion auth when supported.** `APPLICATIONINSIGHTS_CONNECTION_STRING` identifies the target Application Insights resource, and `credential=DefaultAzureCredential(...)` provides Microsoft Entra authentication. +> - Local dev: `DefaultAzureCredential` works as-is. +> - Production: set `AZURE_TOKEN_CREDENTIALS=prod` (or `AZURE_TOKEN_CREDENTIALS=`) to constrain the credential chain to production-safe credentials. +> 2. **Providers are not context managers.** Flush and shut down telemetry providers explicitly at process exit so buffers are exported deterministically. +> +> Snippets may abbreviate this setup, but production code should always follow both rules. ## Quick Start @@ -236,7 +245,7 @@ configure_azure_monitor( ## Best Practices 1. **Pick sync OR async and stay consistent.** Do not mix `azure.xxx` sync clients with `azure.xxx.aio` async clients in the same call path. Choose one mode per module. -2. **Flush and shut down providers at process exit.** Call the shutdown/flush APIs (e.g. `tracer_provider.shutdown()`, `meter_provider.shutdown()`, `logger_provider.shutdown()`) at process exit to flush telemetry before the process terminates. +2. **Call `provider.shutdown()` / `flush()` at process exit to flush telemetry — providers are not context managers.** 3. **Call configure_azure_monitor() early** — Before importing instrumented libraries 4. **Use environment variables** for connection string in production 5. **Set cloud role name** for multi-service applications @@ -244,3 +253,10 @@ configure_azure_monitor( 7. **Use structured logging** for better log analytics queries 8. **Add custom attributes** to spans for better debugging 9. **Use Microsoft Entra authentication** for production workloads + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/capabilities.md new file mode 100644 index 00000000..8876f7ae --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/capabilities.md @@ -0,0 +1,48 @@ +# azure-monitor-opentelemetry-py capability coverage + +**SDK/package**: `azure-monitor-opentelemetry` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Explicit Configuration` +- `With Flask` +- `With Django` +- `With FastAPI` + +## Non-hero scenarios + +- `Custom Traces`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#custom-traces`](non-hero-scenarios.md#custom-traces) +- `Custom Metrics`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#custom-metrics`](non-hero-scenarios.md#custom-metrics) +- `Custom Logs`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#custom-logs`](non-hero-scenarios.md#custom-logs) +- `Sampling`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#sampling`](non-hero-scenarios.md#sampling) +- `Cloud Role Name`: Set cloud role name for Application Map: + See: [`non-hero-scenarios.md#cloud-role-name`](non-hero-scenarios.md#cloud-role-name) +- `Disable Specific Instrumentations`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#disable-specific-instrumentations`](non-hero-scenarios.md#disable-specific-instrumentations) +- `Enable Live Metrics`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#enable-live-metrics`](non-hero-scenarios.md#enable-live-metrics) +- `Azure AD Authentication`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#azure-ad-authentication`](non-hero-scenarios.md#azure-ad-authentication) +- `Auto-Instrumentations Included`: | Library | Telemetry Type | + See: [`non-hero-scenarios.md#auto-instrumentations-included`](non-hero-scenarios.md#auto-instrumentations-included) +- `Configuration Options`: | Parameter | Description | Default | + See: [`non-hero-scenarios.md#configuration-options`](non-hero-scenarios.md#configuration-options) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..384bb1c2 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py/references/non-hero-scenarios.md @@ -0,0 +1,136 @@ +# azure-monitor-opentelemetry-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Custom Traces + +```python +from opentelemetry import trace +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor() + +tracer = trace.get_tracer(__name__) + +with tracer.start_as_current_span("my-operation") as span: + span.set_attribute("custom.attribute", "value") + # Do work... +``` + +## Custom Metrics + +```python +from opentelemetry import metrics +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor() + +meter = metrics.get_meter(__name__) +counter = meter.create_counter("my_counter") + +counter.add(1, {"dimension": "value"}) +``` + +## Custom Logs + +```python +import logging +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor() + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +logger.info("This will appear in Application Insights") +logger.error("Errors are captured too", exc_info=True) +``` + +## Sampling + +```python +from azure.monitor.opentelemetry import configure_azure_monitor + +# Sample 10% of requests +configure_azure_monitor( + sampling_ratio=0.1 +) +``` + +## Cloud Role Name + +Set cloud role name for Application Map: + +```python +from azure.monitor.opentelemetry import configure_azure_monitor +from opentelemetry.sdk.resources import Resource, SERVICE_NAME + +configure_azure_monitor( + resource=Resource.create({SERVICE_NAME: "my-service-name"}) +) +``` + +## Disable Specific Instrumentations + +```python +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor( + instrumentations=["flask", "requests"] # Only enable these +) +``` + +## Enable Live Metrics + +```python +from azure.monitor.opentelemetry import configure_azure_monitor + +configure_azure_monitor( + enable_live_metrics=True +) +``` + +## Azure AD Authentication + +```python +from azure.monitor.opentelemetry import configure_azure_monitor +from azure.identity import DefaultAzureCredential, ManagedIdentityCredential + +# Local dev: DefaultAzureCredential. Production: set AZURE_TOKEN_CREDENTIALS=prod or AZURE_TOKEN_CREDENTIALS= +credential = DefaultAzureCredential(require_envvar=True) +# Or use a specific credential directly in production: +# See https://learn.microsoft.com/python/api/overview/azure/identity-readme?view=azure-python#credential-classes +# credential = ManagedIdentityCredential() + +configure_azure_monitor( + credential=credential +) +``` + +## Auto-Instrumentations Included + +| Library | Telemetry Type | +|---------|---------------| +| Flask | Traces | +| Django | Traces | +| FastAPI | Traces | +| Requests | Traces | +| urllib3 | Traces | +| httpx | Traces | +| aiohttp | Traces | +| psycopg2 | Traces | +| pymysql | Traces | +| pymongo | Traces | +| redis | Traces | + +## Configuration Options + +| Parameter | Description | Default | +|-----------|-------------|---------| +| `connection_string` | Application Insights connection string | From env var | +| `credential` | Azure credential for AAD auth | None | +| `sampling_ratio` | Sampling rate (0.0 to 1.0) | 1.0 | +| `resource` | OpenTelemetry Resource | Auto-detected | +| `instrumentations` | List of instrumentations to enable | All | +| `enable_live_metrics` | Enable Live Metrics stream | False | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/SKILL.md index 95defd74..c0ce6826 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/SKILL.md @@ -267,3 +267,10 @@ AppExceptions 7. **Convert to DataFrame** for easier data analysis 8. **Use aggregations** to summarize metric data 9. **Filter by dimensions** to narrow metric results + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/capabilities.md new file mode 100644 index 00000000..8cf8156e --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/capabilities.md @@ -0,0 +1,30 @@ +# azure-monitor-query-py capability coverage + +**SDK/package**: `azure-monitor-query` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Logs Query Client` +- `Metrics Query Client` +- `Async Clients` +- `Common Kusto Queries` + +## Non-hero scenarios + +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..82a76632 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-monitor-query-py/references/non-hero-scenarios.md @@ -0,0 +1,11 @@ +# azure-monitor-query-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Client Types + +| Client | Purpose | +|--------|---------| +| `LogsQueryClient` | Query Log Analytics workspaces | +| `MetricsQueryClient` | Query Azure Monitor metrics | diff --git a/.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py/SKILL.md index b0f96a3e..9fa73c65 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py/SKILL.md @@ -37,6 +37,17 @@ AZURE_SPEECH_ENDPOINT=https://.stt.speech.microsoft.com pip install requests ``` +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **This API currently requires key-based authentication.** `DefaultAzureCredential` / Entra ID auth isn't available for this REST path yet, so use `AZURE_SPEECH_KEY` and `AZURE_SPEECH_REGION` from environment variables (never hardcoded in source). +> 2. **Use context managers for files and HTTP resources** so file handles and network connections are released deterministically: +> - Sync: `with open(...) as f:` and (when reusing connections) `with requests.Session() as session:` +> - Async: `async with aiohttp.ClientSession() as session:` +> +> Snippets may abbreviate this setup, but production code should always follow both rules. + ## Quick Start ```python @@ -352,7 +363,7 @@ Common language codes (see [full list](https://learn.microsoft.com/azure/ai-serv ## Best Practices 1. **Pick sync OR async and stay consistent.** Do not mix `azure.xxx` sync clients with `azure.xxx.aio` async clients in the same call path. Choose one mode per module. -2. **Always use context managers for clients.** Use `with httpx.Client(...) as client:` (sync) or `async with httpx.AsyncClient(...) as client:` (async) so connections are pooled and closed deterministically. +2. **Use context managers for files and HTTP resources.** Use `with open(...) as f:` and (when reusing connections) `with requests.Session() as session:` for sync code, or `async with aiohttp.ClientSession() as session:` for async code. 3. **Use WAV PCM 16kHz mono** for best compatibility 4. **Enable chunked transfer** for lower latency 5. **Cache access tokens** for 9 minutes (valid for 10) diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/SKILL.md index 45cb8391..31cd68bb 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/SKILL.md @@ -255,3 +255,10 @@ async def download_async(): 6. **Prefer `readinto()`** over `readall()` for memory efficiency 7. **Use `walk_blobs()`** for hierarchical listing 8. **Set appropriate content types** for web-served blobs + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Capability index mapping hero flows and non-hero references. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples (metadata/properties and async patterns). | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/capabilities.md new file mode 100644 index 00000000..3a153a14 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/capabilities.md @@ -0,0 +1,32 @@ +# azure-storage-blob-py capability coverage + +**SDK/package**: `azure-storage-blob` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Client Hierarchy` +- `Core Workflow` +- `Performance Tuning` +- `SAS Tokens (User Delegation)` + +## Non-hero scenarios + +- `Blob Properties and Metadata`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#blob-properties-and-metadata`](non-hero-scenarios.md#blob-properties-and-metadata) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..8952d21a --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-blob-py/references/non-hero-scenarios.md @@ -0,0 +1,46 @@ +# azure-storage-blob-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Blob Properties and Metadata + +```python +# Get properties +properties = blob_client.get_blob_properties() +print(f"Size: {properties.size}") +print(f"Content-Type: {properties.content_settings.content_type}") +print(f"Last modified: {properties.last_modified}") + +# Set metadata +blob_client.set_blob_metadata(metadata={"category": "logs", "year": "2024"}) + +# Set content type +from azure.storage.blob import ContentSettings +blob_client.set_http_headers( + content_settings=ContentSettings(content_type="application/json") +) +``` + +## Async Client + +```python +from azure.identity.aio import DefaultAzureCredential +from azure.storage.blob.aio import BlobServiceClient + +async def upload_async(): + async with DefaultAzureCredential() as credential: + async with BlobServiceClient(account_url, credential=credential) as client: + blob_client = client.get_blob_client("mycontainer", "sample.txt") + + with open("./file.txt", "rb") as data: + await blob_client.upload_blob(data, overwrite=True) + +# Download async +async def download_async(): + async with BlobServiceClient(account_url, credential=credential) as client: + blob_client = client.get_blob_client("mycontainer", "sample.txt") + + stream = await blob_client.download_blob() + data = await stream.readall() +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/SKILL.md index cbf9de12..bd8a9e4a 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/SKILL.md @@ -233,3 +233,10 @@ asyncio.run(datalake_operations()) 8. **Use `get_paths` with `recursive=True`** for full directory listing 9. **Set metadata** for custom file attributes 10. **Consider Blob API** for simple object storage use cases + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/capabilities.md new file mode 100644 index 00000000..b1d10159 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/capabilities.md @@ -0,0 +1,36 @@ +# azure-storage-file-datalake-py capability coverage + +**SDK/package**: `azure-storage-file-datalake` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Client Hierarchy` +- `File System Operations` +- `Directory Operations` +- `File Operations` + +## Non-hero scenarios + +- `List Contents`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#list-contents`](non-hero-scenarios.md#list-contents) +- `File/Directory Properties`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#filedirectory-properties`](non-hero-scenarios.md#filedirectory-properties) +- `Access Control (ACL)`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#access-control-acl`](non-hero-scenarios.md#access-control-acl) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..f23a4960 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py/references/non-hero-scenarios.md @@ -0,0 +1,77 @@ +# azure-storage-file-datalake-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## List Contents + +```python +# List paths (files and directories) +for path in file_system_client.get_paths(): + print(f"{'DIR' if path.is_directory else 'FILE'}: {path.name}") + +# List paths in directory +for path in file_system_client.get_paths(path="mydir"): + print(path.name) + +# Recursive listing +for path in file_system_client.get_paths(path="mydir", recursive=True): + print(path.name) +``` + +## File/Directory Properties + +```python +# Get properties +properties = file_client.get_file_properties() +print(f"Size: {properties.size}") +print(f"Last modified: {properties.last_modified}") + +# Set metadata +file_client.set_metadata(metadata={"processed": "true"}) +``` + +## Access Control (ACL) + +```python +# Get ACL +acl = directory_client.get_access_control() +print(f"Owner: {acl['owner']}") +print(f"Permissions: {acl['permissions']}") + +# Set ACL +directory_client.set_access_control( + owner="user-id", + permissions="rwxr-x---" +) + +# Update ACL entries +from azure.storage.filedatalake import AccessControlChangeResult +directory_client.update_access_control_recursive( + acl="user:user-id:rwx" +) +``` + +## Async Client + +```python +from azure.storage.filedatalake.aio import DataLakeServiceClient +from azure.identity.aio import DefaultAzureCredential + +async def datalake_operations(): + async with DefaultAzureCredential() as credential: + async with DataLakeServiceClient( + account_url="https://.dfs.core.windows.net", + credential=credential + ) as service_client: + file_system_client = service_client.get_file_system_client("myfilesystem") + file_client = file_system_client.get_file_client("test.txt") + + await file_client.upload_data(b"async content", overwrite=True) + + download = await file_client.download_file() + content = await download.readall() + +import asyncio +asyncio.run(datalake_operations()) +``` diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/SKILL.md index 5200889a..5adf7ad6 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/SKILL.md @@ -243,3 +243,10 @@ async def upload_file(): 6. **Create snapshots** before major changes 7. **Set quotas** to prevent unexpected storage costs 8. **Use ranges** for partial file updates + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/capabilities.md new file mode 100644 index 00000000..6044d9b1 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/capabilities.md @@ -0,0 +1,34 @@ +# azure-storage-file-share-py capability coverage + +**SDK/package**: `azure-storage-file-share` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Share Operations` +- `Directory Operations` +- `File Operations` +- `Range Operations` + +## Non-hero scenarios + +- `Snapshot Operations`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#snapshot-operations`](non-hero-scenarios.md#snapshot-operations) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Client Types`: | Client | Purpose | + See: [`non-hero-scenarios.md#client-types`](non-hero-scenarios.md#client-types) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..e43eab1f --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py/references/non-hero-scenarios.md @@ -0,0 +1,46 @@ +# azure-storage-file-share-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Snapshot Operations + +### Create Snapshot + +```python +snapshot = share_client.create_snapshot() +print(f"Snapshot: {snapshot['snapshot']}") +``` + +### Access Snapshot + +```python +snapshot_client = service.get_share_client( + "my-share", + snapshot=snapshot["snapshot"] +) +``` + +## Async Client + +```python +from azure.storage.fileshare.aio import ShareServiceClient +from azure.identity.aio import DefaultAzureCredential + +async def upload_file(): + async with DefaultAzureCredential() as credential: + async with ShareServiceClient(account_url, credential=credential) as service: + share = service.get_share_client("my-share") + file_client = share.get_file_client("test.txt") + + await file_client.upload_file("Hello!") +``` + +## Client Types + +| Client | Purpose | +|--------|---------| +| `ShareServiceClient` | Account-level operations | +| `ShareClient` | Share operations | +| `ShareDirectoryClient` | Directory operations | +| `ShareFileClient` | File operations | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/SKILL.md index 2e953c26..f8ed9816 100644 --- a/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/SKILL.md @@ -237,3 +237,10 @@ with QueueClient( 8. **Use `peek_messages`** for monitoring without affecting queue 9. **Set `time_to_live`** to prevent stale messages 10. **Consider Service Bus** for advanced features (sessions, topics) + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Dedicated non-hero examples for secondary/advanced scenarios. | diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/capabilities.md new file mode 100644 index 00000000..af6e9c29 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/capabilities.md @@ -0,0 +1,40 @@ +# azure-storage-queue-py capability coverage + +**SDK/package**: `azure-storage-queue` + +This index maps hero scenarios in `SKILL.md` and links non-hero scenarios documented in dedicated reference files. + +## Hero scenarios covered in SKILL.md + +- `Queue Operations` +- `Send Messages` +- `Receive Messages` +- `Peek Messages` + +## Non-hero scenarios + +- `Update Message`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#update-message`](non-hero-scenarios.md#update-message) +- `Delete Message`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#delete-message`](non-hero-scenarios.md#delete-message) +- `Clear Queue`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#clear-queue`](non-hero-scenarios.md#clear-queue) +- `Queue Properties`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#queue-properties`](non-hero-scenarios.md#queue-properties) +- `Async Client`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#async-client`](non-hero-scenarios.md#async-client) +- `Base64 Encoding`: Dedicated example and implementation notes. + See: [`non-hero-scenarios.md#base64-encoding`](non-hero-scenarios.md#base64-encoding) + +## Related deep-dive references + +- [`non-hero-scenarios.md`](non-hero-scenarios.md): Dedicated non-hero examples and implementation notes. + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- For Python SDKs with both sync and async clients, document both forms without a blanket preference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/non-hero-scenarios.md b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/non-hero-scenarios.md new file mode 100644 index 00000000..2d595015 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/azure-storage-queue-py/references/non-hero-scenarios.md @@ -0,0 +1,101 @@ +# azure-storage-queue-py non-hero scenarios + +These scenarios are intentionally separate from hero flows in `SKILL.md`. +They cover secondary/advanced patterns typically used after the primary end-to-end path is working. + +## Update Message + +```python +# Extend visibility or update content +messages = queue_client.receive_messages() +for message in messages: + # Extend timeout (need more time) + queue_client.update_message( + message, + visibility_timeout=60 + ) + + # Update content and timeout + queue_client.update_message( + message, + content="Updated content", + visibility_timeout=60 + ) +``` + +## Delete Message + +```python +# Delete after successful processing +messages = queue_client.receive_messages() +for message in messages: + try: + # Process... + queue_client.delete_message(message) + except Exception: + # Message becomes visible again after timeout + pass +``` + +## Clear Queue + +```python +# Delete all messages +queue_client.clear_messages() +``` + +## Queue Properties + +```python +# Get queue properties +properties = queue_client.get_queue_properties() +print(f"Approximate message count: {properties.approximate_message_count}") + +# Set/get metadata +queue_client.set_queue_metadata(metadata={"environment": "production"}) +properties = queue_client.get_queue_properties() +print(properties.metadata) +``` + +## Async Client + +```python +from azure.storage.queue.aio import QueueServiceClient, QueueClient +from azure.identity.aio import DefaultAzureCredential + +async def queue_operations(): + credential = DefaultAzureCredential() + + async with QueueClient( + account_url="https://.queue.core.windows.net", + queue_name="myqueue", + credential=credential + ) as client: + # Send + await client.send_message("Async message") + + # Receive + async for message in client.receive_messages(): + print(message.content) + await client.delete_message(message) + +import asyncio +asyncio.run(queue_operations()) +``` + +## Base64 Encoding + +```python +from azure.storage.queue import QueueClient, BinaryBase64EncodePolicy, BinaryBase64DecodePolicy + +# For binary data +with QueueClient( + account_url=account_url, + queue_name="myqueue", + credential=credential, + message_encode_policy=BinaryBase64EncodePolicy(), + message_decode_policy=BinaryBase64DecodePolicy() +) as queue_client: + # Send bytes + queue_client.send_message(b"Binary content") +``` diff --git a/.github/plugins/azure-sdk-python/skills/fastapi-router-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/fastapi-router-py/SKILL.md index 0d6f32e6..1e966fef 100644 --- a/.github/plugins/azure-sdk-python/skills/fastapi-router-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/fastapi-router-py/SKILL.md @@ -57,5 +57,11 @@ async def list_items() -> list[Item]: ## Best Practices -1. **Pick `def` or `async def` per endpoint based on whether you call async I/O;** do not mix sync and async blocking calls in one handler. +1. **Pick `def` or `async def` per endpoint based on whether you call async I/O; do not mix sync and blocking calls in one handler.** 2. **Manage long-lived resources (DB pools, HTTP clients) in `lifespan` and inject via `Depends`;** use `with`/`async with` for per-request resources. + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | diff --git a/.github/plugins/azure-sdk-python/skills/fastapi-router-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/fastapi-router-py/references/capabilities.md new file mode 100644 index 00000000..ba5506a4 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/fastapi-router-py/references/capabilities.md @@ -0,0 +1,25 @@ +# fastapi-router-py capability coverage + +**SDK/package**: `fastapi-router-py` + +This reference captures additional non-hero capabilities and API breadth so the main `SKILL.md` can stay focused on copy/paste hero flows. + +## Hero scenarios covered in SKILL.md + +- `Quick Start` +- `Authentication Patterns` +- `Response Models` +- `HTTP Status Codes` + +## Important non-hero scenarios to include when needed + +- `Integration Steps` +- `Best Practices` + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/m365-agents-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/m365-agents-py/SKILL.md index 743e9ac1..3d583a89 100644 --- a/.github/plugins/azure-sdk-python/skills/m365-agents-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/m365-agents-py/SKILL.md @@ -54,6 +54,15 @@ COPILOTSTUDIOAGENT__TENANTID= COPILOTSTUDIOAGENT__AGENTAPPID= ``` +## Authentication & Lifecycle + +> **🔑 Two rules apply to every code sample below:** +> +> 1. **This SDK is async-first — use `async def` handlers and `async with` throughout.** +> 2. **Use explicit auth managers and context-managed network resources.** Use `MsalConnectionManager` for agent auth, and wrap per-request HTTP resources in `async with` (for example, `aiohttp.ClientSession`). +> +> Snippets may abbreviate this setup, but production code should always follow both rules. + ## Core Workflow: aiohttp-hosted AgentApplication ```python @@ -333,7 +342,7 @@ asyncio.run(main()) ## Best Practices -1. **This skill is async-first (aiohttp-based).** Use async handlers and `async with` for aiohttp sessions. +1. **This SDK is async-first — use `async def` handlers and `async with` throughout.** 2. **Always use context managers for clients and async credentials.** Wrap every client in `with Client(...) as client:` (sync) or `async with Client(...) as client:` (async). For async `DefaultAzureCredential` from `azure.identity.aio`, also use `async with credential:` so tokens and transports are cleaned up. 3. Use `microsoft_agents` import prefix (underscores, not dots). 4. Use `MemoryStorage` only for development; use BlobStorage or CosmosDB in production. @@ -352,3 +361,9 @@ asyncio.run(main()) | GitHub samples (Python) | https://github.com/microsoft/Agents-for-python | | PyPI packages | https://pypi.org/search/?q=microsoft-agents | | Integrate with Copilot Studio | https://learn.microsoft.com/en-us/microsoft-365/agents-sdk/integrate-with-mcs | + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | diff --git a/.github/plugins/azure-sdk-python/skills/m365-agents-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/m365-agents-py/references/capabilities.md new file mode 100644 index 00000000..aedf8e7c --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/m365-agents-py/references/capabilities.md @@ -0,0 +1,42 @@ +# m365-agents-py capability coverage + +**SDK/package**: `microsoft-agents-hosting-core, microsoft-agents-hosting-aiohttp, microsoft-agents-activity, microsoft-agents-authentication-msal, microsoft-agents-copilotstudio-client` + +This reference mirrors the actual capability sections in `SKILL.md` and provides concrete non-hero examples for implementation guidance. + +## Hero scenarios covered in SKILL.md + +- `Core Workflow: aiohttp-hosted AgentApplication` +- `AgentApplication Routing` +- `Streaming Responses with Azure OpenAI` +- `OAuth / Auto Sign-In` + +## Important non-hero scenarios with examples + +### `Copilot Studio Client (Direct to Engine)` + +```python +import asyncio +from msal import PublicClientApplication +from microsoft_agents.activity import ActivityTypes, load_configuration_from_env +from microsoft_agents.copilotstudio.client import ( + ConnectionSettings, + CopilotClient, +) + +# Token cache (local file for interactive flows) +class LocalTokenCache: + # See samples for full implementation + pass + +def acquire_token(settings, app_client_id, tenant_id): +# ... see SKILL.md for the full example +``` + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-python/skills/pydantic-models-py/SKILL.md b/.github/plugins/azure-sdk-python/skills/pydantic-models-py/SKILL.md index 56de5e40..2605b360 100644 --- a/.github/plugins/azure-sdk-python/skills/pydantic-models-py/SKILL.md +++ b/.github/plugins/azure-sdk-python/skills/pydantic-models-py/SKILL.md @@ -60,3 +60,9 @@ class MyInDB(MyResponse): 1. Create models in `src/backend/app/models/` 2. Export from `src/backend/app/models/__init__.py` 3. Add corresponding TypeScript types + +## Reference Files + +| File | Contents | +|------|----------| +| [references/capabilities.md](references/capabilities.md) | Additional non-hero capabilities, operation-group coverage, and production checklists. | diff --git a/.github/plugins/azure-sdk-python/skills/pydantic-models-py/references/capabilities.md b/.github/plugins/azure-sdk-python/skills/pydantic-models-py/references/capabilities.md new file mode 100644 index 00000000..5ed18ff5 --- /dev/null +++ b/.github/plugins/azure-sdk-python/skills/pydantic-models-py/references/capabilities.md @@ -0,0 +1,25 @@ +# pydantic-models-py capability coverage + +**SDK/package**: `pydantic-models-py` + +This reference captures additional non-hero capabilities and API breadth so the main `SKILL.md` can stay focused on copy/paste hero flows. + +## Hero scenarios covered in SKILL.md + +- `Quick Start` +- `Multi-Model Pattern` +- `camelCase Aliases` +- `Optional Update Fields` + +## Important non-hero scenarios to include when needed + +- `Database Document` +- `Integration Steps` + +## API breadth checklist + +- Verify client/auth mode for the environment before coding. +- Confirm operation-group/method names against current Microsoft Learn API reference. +- Include cleanup/delete paths for created resources in examples. +- Prefer idempotent create/update operations where available. +- Validate paging/LRO/error-handling patterns for production paths. diff --git a/.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust/SKILL.md b/.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust/SKILL.md deleted file mode 100644 index 365a93a6..00000000 --- a/.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust/SKILL.md +++ /dev/null @@ -1,171 +0,0 @@ ---- -name: azure-servicebus-rust -description: | - Azure Service Bus library for Rust. Send and receive messages using queues, topics, and subscriptions. - Triggers: "service bus rust", "ServiceBusClient rust", "send message servicebus rust", "receive message servicebus rust", "queue rust messaging", "topic subscription rust". -license: MIT -metadata: - author: Microsoft - package: azure_messaging_servicebus ---- - -# Azure Service Bus library for Rust - -Client library for Azure Service Bus — enterprise message broker with queues and publish-subscribe topics. - -> **⚠️ WARNING:** This crate is in early development and **SHOULD NOT** be used in production. APIs may change without notice. - -Use this skill when: - -- An app needs to send or receive messages via Azure Service Bus from Rust -- You need queue-based messaging with competing consumers -- You need publish-subscribe messaging with topics and subscriptions -- You need reliable message delivery with completion semantics - -> **IMPORTANT:** Only use the official `azure_messaging_servicebus` crate published by the [azure-sdk](https://crates.io/users/azure-sdk) crates.io user. Do NOT use unofficial or community crates. Official crates use underscores in names and none have version 0.21.0. - -## Installation - -```sh -cargo add azure_messaging_servicebus azure_identity tokio -``` - -> If your code uses `azure_core` types directly, add `azure_core` to `Cargo.toml`. If you only use `azure_messaging_servicebus` re-exports, direct `azure_core` dependency is optional. - -## Environment Variables - -```bash -SERVICEBUS_NAMESPACE=.servicebus.windows.net # Required — fully qualified namespace -``` - -## Key Concepts - -| Concept | Description | -| ---------------- | --------------------------------------------------------------- | -| **Namespace** | Container for all messaging components | -| **Queue** | Point-to-point messaging with competing consumers | -| **Topic** | Publish-subscribe messaging — one sender, many subscribers | -| **Subscription** | Receives messages from a topic | -| **Message** | Package of data and metadata, with completion/abandon semantics | - -## Authentication - -```rust -use azure_identity::DeveloperToolsCredential; -use azure_messaging_servicebus::ServiceBusClient; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Local dev: DeveloperToolsCredential. Production: use ManagedIdentityCredential. - let credential = DeveloperToolsCredential::new(None)?; - let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; - Ok(()) -} -``` - -## Core Workflow - -### Send a Message to a Queue - -```rust -use azure_identity::DeveloperToolsCredential; -use azure_messaging_servicebus::{ServiceBusClient, Message}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - let credential = DeveloperToolsCredential::new(None)?; - let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; - let sender = client.create_sender("my_queue", None).await?; - - let message = Message::from("Hello, Service Bus!"); - sender.send_message(message, None).await?; - Ok(()) -} -``` - -### Receive Messages from a Queue - -```rust -use azure_identity::DeveloperToolsCredential; -use azure_messaging_servicebus::ServiceBusClient; - -#[tokio::main] -async fn main() -> Result<(), Box> { - let credential = DeveloperToolsCredential::new(None)?; - let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; - let receiver = client.create_receiver("my_queue", None).await?; - - let messages = receiver.receive_messages(5, None).await?; - for message in messages { - println!("Received: {}", message.body_as_string()?); - receiver.complete_message(&message, None).await?; - } - Ok(()) -} -``` - -### Send a Message to a Topic - -```rust -let sender = client.create_sender("my_topic", None).await?; -let message = Message::from("Hello, Topic subscribers!"); -sender.send_message(message, None).await?; -``` - -### Receive Messages from a Subscription - -```rust -let receiver = client - .create_receiver_for_subscription("my_topic", "my_subscription", None) - .await?; - -let messages = receiver.receive_messages(5, None).await?; -for message in messages { - println!("Received: {}", message.body_as_string()?); - receiver.complete_message(&message, None).await?; -} -``` - -## Message Settlement - -| Action | Purpose | -| ---------- | -------------------------------------------------- | -| `complete` | Remove message from queue — processing succeeded | -| `abandon` | Release lock — message becomes available for retry | - -Always complete messages after successful processing to prevent redelivery. - -## RBAC Roles - -For Entra ID auth, assign one of these roles: - -| Role | Access | -| --------------------------------- | ---------------- | -| `Azure Service Bus Data Sender` | Send messages | -| `Azure Service Bus Data Receiver` | Receive messages | -| `Azure Service Bus Data Owner` | Full access | - -## Best Practices - -1. **Use `cargo add` to manage dependencies, never edit `Cargo.toml` directly.** Add and remove Rust SDK dependencies with cargo commands instead of manual manifest edits. -2. **Add `azure_core` only when importing `azure_core` types directly.** If your code imports `azure_core::http::Url`, `azure_core::http::RequestContent`, or `azure_core::error::ErrorKind`, include `azure_core`; otherwise a direct dependency is optional. -3. **Use `DeveloperToolsCredential`** for local dev, **`ManagedIdentityCredential`** for production — Rust does not provide a single `DefaultAzureCredential` type -4. **Never hardcode credentials** — use environment variables or managed identity -5. **Assign RBAC roles** — ensure the identity has appropriate Service Bus data roles -6. **Always complete messages** — call `complete_message` after processing to remove from queue -7. **Use topics for fan-out** — when multiple consumers need the same messages, use topics with subscriptions -8. **This crate is pre-production** — APIs may change; pin your dependency version with cargo commands in your dependency workflow - -## Reference Links - -| Resource | Link | -| ------------- | ----------------------------------------------------------------------------------------------- | -| API Reference | https://docs.rs/azure_messaging_servicebus/latest/azure_messaging_servicebus | -| crates.io | https://crates.io/crates/azure_messaging_servicebus | -| Source Code | https://github.com/Azure/azure-sdk-for-rust/tree/main/sdk/servicebus/azure_messaging_servicebus | diff --git a/.github/skills/agent-framework-azure-ai-py b/.github/skills/agent-framework-azure-ai-py new file mode 120000 index 00000000..0f574683 --- /dev/null +++ b/.github/skills/agent-framework-azure-ai-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/agent-framework-azure-ai-py \ No newline at end of file diff --git a/.github/skills/azure-ai-contentsafety-py b/.github/skills/azure-ai-contentsafety-py new file mode 120000 index 00000000..08ab0c46 --- /dev/null +++ b/.github/skills/azure-ai-contentsafety-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-contentsafety-py \ No newline at end of file diff --git a/.github/skills/azure-ai-contentunderstanding-py b/.github/skills/azure-ai-contentunderstanding-py new file mode 120000 index 00000000..8d4a2e04 --- /dev/null +++ b/.github/skills/azure-ai-contentunderstanding-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py \ No newline at end of file diff --git a/.github/skills/azure-ai-language-conversations-py b/.github/skills/azure-ai-language-conversations-py new file mode 120000 index 00000000..257a9470 --- /dev/null +++ b/.github/skills/azure-ai-language-conversations-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-language-conversations-py \ No newline at end of file diff --git a/.github/skills/azure-ai-ml-py b/.github/skills/azure-ai-ml-py new file mode 120000 index 00000000..eaac1e82 --- /dev/null +++ b/.github/skills/azure-ai-ml-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-ml-py \ No newline at end of file diff --git a/.github/skills/azure-ai-projects-py b/.github/skills/azure-ai-projects-py new file mode 120000 index 00000000..eaaf98bc --- /dev/null +++ b/.github/skills/azure-ai-projects-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-projects-py \ No newline at end of file diff --git a/.github/skills/azure-ai-textanalytics-py b/.github/skills/azure-ai-textanalytics-py new file mode 120000 index 00000000..650baacf --- /dev/null +++ b/.github/skills/azure-ai-textanalytics-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-textanalytics-py \ No newline at end of file diff --git a/.github/skills/azure-ai-transcription-py b/.github/skills/azure-ai-transcription-py new file mode 120000 index 00000000..f5b70e97 --- /dev/null +++ b/.github/skills/azure-ai-transcription-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-transcription-py \ No newline at end of file diff --git a/.github/skills/azure-ai-translation-document-py b/.github/skills/azure-ai-translation-document-py new file mode 120000 index 00000000..d81c331a --- /dev/null +++ b/.github/skills/azure-ai-translation-document-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-translation-document-py \ No newline at end of file diff --git a/.github/skills/azure-ai-translation-text-py b/.github/skills/azure-ai-translation-text-py new file mode 120000 index 00000000..a0d31c26 --- /dev/null +++ b/.github/skills/azure-ai-translation-text-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-translation-text-py \ No newline at end of file diff --git a/.github/skills/azure-ai-vision-imageanalysis-py b/.github/skills/azure-ai-vision-imageanalysis-py new file mode 120000 index 00000000..0cf02c59 --- /dev/null +++ b/.github/skills/azure-ai-vision-imageanalysis-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py \ No newline at end of file diff --git a/.github/skills/azure-ai-voicelive-py b/.github/skills/azure-ai-voicelive-py new file mode 120000 index 00000000..ee0d9929 --- /dev/null +++ b/.github/skills/azure-ai-voicelive-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-ai-voicelive-py \ No newline at end of file diff --git a/.github/skills/azure-appconfiguration-py b/.github/skills/azure-appconfiguration-py new file mode 120000 index 00000000..0c8182d0 --- /dev/null +++ b/.github/skills/azure-appconfiguration-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-appconfiguration-py \ No newline at end of file diff --git a/.github/skills/azure-containerregistry-py b/.github/skills/azure-containerregistry-py new file mode 120000 index 00000000..ee9c2e53 --- /dev/null +++ b/.github/skills/azure-containerregistry-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-containerregistry-py \ No newline at end of file diff --git a/.github/skills/azure-cosmos-db-py b/.github/skills/azure-cosmos-db-py new file mode 120000 index 00000000..4b58584f --- /dev/null +++ b/.github/skills/azure-cosmos-db-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-cosmos-db-py \ No newline at end of file diff --git a/.github/skills/azure-cosmos-py b/.github/skills/azure-cosmos-py new file mode 120000 index 00000000..ab82f04f --- /dev/null +++ b/.github/skills/azure-cosmos-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-cosmos-py \ No newline at end of file diff --git a/.github/skills/azure-data-tables-py b/.github/skills/azure-data-tables-py new file mode 120000 index 00000000..a2dbba4d --- /dev/null +++ b/.github/skills/azure-data-tables-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-data-tables-py \ No newline at end of file diff --git a/.github/skills/azure-eventgrid-py b/.github/skills/azure-eventgrid-py new file mode 120000 index 00000000..d3caa828 --- /dev/null +++ b/.github/skills/azure-eventgrid-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-eventgrid-py \ No newline at end of file diff --git a/.github/skills/azure-eventhub-py b/.github/skills/azure-eventhub-py new file mode 120000 index 00000000..d2fc8221 --- /dev/null +++ b/.github/skills/azure-eventhub-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-eventhub-py \ No newline at end of file diff --git a/.github/skills/azure-identity-py b/.github/skills/azure-identity-py new file mode 120000 index 00000000..dfe20c2b --- /dev/null +++ b/.github/skills/azure-identity-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-identity-py \ No newline at end of file diff --git a/.github/skills/azure-keyvault-py b/.github/skills/azure-keyvault-py new file mode 120000 index 00000000..2335eac7 --- /dev/null +++ b/.github/skills/azure-keyvault-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-keyvault-py \ No newline at end of file diff --git a/.github/skills/azure-messaging-webpubsubservice-py b/.github/skills/azure-messaging-webpubsubservice-py new file mode 120000 index 00000000..41368c4a --- /dev/null +++ b/.github/skills/azure-messaging-webpubsubservice-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-apicenter-py b/.github/skills/azure-mgmt-apicenter-py new file mode 120000 index 00000000..446781ba --- /dev/null +++ b/.github/skills/azure-mgmt-apicenter-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-apimanagement-py b/.github/skills/azure-mgmt-apimanagement-py new file mode 120000 index 00000000..adf8958c --- /dev/null +++ b/.github/skills/azure-mgmt-apimanagement-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-botservice-py b/.github/skills/azure-mgmt-botservice-py new file mode 120000 index 00000000..84a3ac19 --- /dev/null +++ b/.github/skills/azure-mgmt-botservice-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-botservice-py \ No newline at end of file diff --git a/.github/skills/azure-mgmt-fabric-py b/.github/skills/azure-mgmt-fabric-py new file mode 120000 index 00000000..67651cbb --- /dev/null +++ b/.github/skills/azure-mgmt-fabric-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-mgmt-fabric-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-ingestion-py b/.github/skills/azure-monitor-ingestion-py new file mode 120000 index 00000000..a31461c2 --- /dev/null +++ b/.github/skills/azure-monitor-ingestion-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-ingestion-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-opentelemetry-exporter-py b/.github/skills/azure-monitor-opentelemetry-exporter-py new file mode 120000 index 00000000..23b34751 --- /dev/null +++ b/.github/skills/azure-monitor-opentelemetry-exporter-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-opentelemetry-py b/.github/skills/azure-monitor-opentelemetry-py new file mode 120000 index 00000000..bcee5c56 --- /dev/null +++ b/.github/skills/azure-monitor-opentelemetry-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py \ No newline at end of file diff --git a/.github/skills/azure-monitor-query-py b/.github/skills/azure-monitor-query-py new file mode 120000 index 00000000..0f8f3bd6 --- /dev/null +++ b/.github/skills/azure-monitor-query-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-monitor-query-py \ No newline at end of file diff --git a/.github/skills/azure-search-documents-py b/.github/skills/azure-search-documents-py new file mode 120000 index 00000000..13a4ec86 --- /dev/null +++ b/.github/skills/azure-search-documents-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-search-documents-py \ No newline at end of file diff --git a/.github/skills/azure-servicebus-py b/.github/skills/azure-servicebus-py new file mode 120000 index 00000000..5275c159 --- /dev/null +++ b/.github/skills/azure-servicebus-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-servicebus-py \ No newline at end of file diff --git a/.github/skills/azure-servicebus-rust b/.github/skills/azure-servicebus-rust deleted file mode 120000 index 898d0443..00000000 --- a/.github/skills/azure-servicebus-rust +++ /dev/null @@ -1 +0,0 @@ -../plugins/azure-sdk-rust/skills/azure-servicebus-rust \ No newline at end of file diff --git a/.github/skills/azure-speech-to-text-rest-py b/.github/skills/azure-speech-to-text-rest-py new file mode 120000 index 00000000..c9bc2f35 --- /dev/null +++ b/.github/skills/azure-speech-to-text-rest-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py \ No newline at end of file diff --git a/.github/skills/azure-storage-blob-py b/.github/skills/azure-storage-blob-py new file mode 120000 index 00000000..6941148a --- /dev/null +++ b/.github/skills/azure-storage-blob-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-blob-py \ No newline at end of file diff --git a/.github/skills/azure-storage-file-datalake-py b/.github/skills/azure-storage-file-datalake-py new file mode 120000 index 00000000..a4f13f7b --- /dev/null +++ b/.github/skills/azure-storage-file-datalake-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-file-datalake-py \ No newline at end of file diff --git a/.github/skills/azure-storage-file-share-py b/.github/skills/azure-storage-file-share-py new file mode 120000 index 00000000..5e1e9b27 --- /dev/null +++ b/.github/skills/azure-storage-file-share-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-file-share-py \ No newline at end of file diff --git a/.github/skills/azure-storage-queue-py b/.github/skills/azure-storage-queue-py new file mode 120000 index 00000000..3260a7ed --- /dev/null +++ b/.github/skills/azure-storage-queue-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/azure-storage-queue-py \ No newline at end of file diff --git a/.github/skills/fastapi-router-py b/.github/skills/fastapi-router-py new file mode 120000 index 00000000..d190edc7 --- /dev/null +++ b/.github/skills/fastapi-router-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/fastapi-router-py \ No newline at end of file diff --git a/.github/skills/m365-agents-py b/.github/skills/m365-agents-py new file mode 120000 index 00000000..a4dddc0e --- /dev/null +++ b/.github/skills/m365-agents-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/m365-agents-py \ No newline at end of file diff --git a/.github/skills/pydantic-models-py b/.github/skills/pydantic-models-py new file mode 120000 index 00000000..5f5f6459 --- /dev/null +++ b/.github/skills/pydantic-models-py @@ -0,0 +1 @@ +../plugins/azure-sdk-python/skills/pydantic-models-py \ No newline at end of file diff --git a/.github/skills/skill-creator/SKILL.md b/.github/skills/skill-creator/SKILL.md index 83773837..337d521f 100644 --- a/.github/skills/skill-creator/SKILL.md +++ b/.github/skills/skill-creator/SKILL.md @@ -89,11 +89,11 @@ For Azure SDK skills, follow the **Skill Section Order** below. For domain skill ### Bundled Resources (Optional) -| Type | When to Include | Examples | -| ------------- | ------------------------- | -------------------------- | -| `scripts/` | Reused code patterns | Auth setup, CLI scripts | -| `references/` | Detailed patterns/schemas | API docs, migration guides | -| `assets/` | Output templates | Boilerplate code, images | +| Type | When to Include | Examples | +| ------------- | ---------------------------------------- | ---------------------------------------------------------- | +| `scripts/` | Reused code patterns | Auth setup, CLI scripts | +| `references/` | Feature deep-dives and overflow examples | `capabilities.md` index, `non-hero-scenarios.md`, API docs | +| `assets/` | Output templates | Boilerplate code, images | --- @@ -101,6 +101,85 @@ For Azure SDK skills, follow the **Skill Section Order** below. For domain skill When creating skills for Azure SDKs, follow these patterns consistently. +### Token Budget Guidelines (REQUIRED) + +Every Azure SDK skill MUST stay within these token limits: + +| Section | Target | Absolute Max | +| ----------------------------- | ---------------- | ---------------- | +| Installation + Env Vars | 100 tokens | 150 | +| Authentication & Lifecycle | 200 tokens | 300 | +| Core Workflow (1 example) | 300 tokens | 400 | +| Feature Tables | 200 tokens | 300 | +| Best Practices (6-8 items) | 200 tokens | 250 | +| References (reference/ links) | 100 tokens | 150 | +| **Total SKILL.md** | **~1100 tokens** | **~1500 tokens** | + +**Enforcement**: + +- Exceeding max limit → refactor into `/references/` subdirectories +- When approaching 500 lines → move entire sections to reference files +- Annotate with `# Token Count: ~XXXX` at top of completed skill + +--- + +### Reference Extraction Guide (REQUIRED) + +Decide what goes in SKILL.md vs. `/references/` using these signals: + +| Signal | Move to `/references/` | Keep in SKILL.md | +| -------------- | ----------------------------------- | ---------------------- | +| Use frequency | <20% of typical use | ~80%+ of workflows | +| Cognitive load | Advanced patterns, multiple options | Single happy path | +| Example length | >10 lines, multiple paths | 1-5 lines, single path | + +**Content extraction rules:** + +- **Batch operations** → `/references/batch-operations.md` +- **Error handling** (beyond try-except) → `/references/error-handling.md` +- **Performance tuning** → `/references/performance.md` +- **Alternative workflows** → `/references/workflows-comparison.md` +- **Streaming/events** → `/references/streaming.md` +- **Advanced auth** → `/references/auth-strategies.md` +- **Tool integration** → `/references/tools.md` +- **Breaking changes** → `/references/migration.md` + +**Decision:** Keep common case in SKILL.md, move edge cases to `/references/`. + +--- + +### Core Workflow Discipline (REQUIRED) + +Every Azure SDK skill must clarify which workflow(s) it documents. + +**Case 1: Single clear "core workflow"** (majority of services) + +If one pattern handles ~80% of use cases: + +1. Designate it as the core workflow +2. Show ONLY this workflow in SKILL.md (one complete, runnable example) +3. Defer alternatives to `/references/`: + - Batch operations → `/references/batch-operations.md` + - Error handling → `/references/error-handling.md` + - Performance tuning → `/references/performance.md` + - Alternative workflows → `/references/other-workflows.md` + +**Example**: Azure Key Vault Secrets (core workflow: retrieve a secret using managed identity). Alternative workflows in `/references/`: API-key auth, certificate auth, service principal auth. + +**Case 2: Multiple equally-valid "core workflows"** (e.g., authentication strategies, deployment targets) + +If no single pattern dominates: + +1. Pick the most common workflow for SKILL.md +2. Acknowledge other approaches in a `/references/workflows-comparison.md` that explains trade-offs (e.g., "Local dev uses DeveloperToolsCredential; production uses ManagedIdentityCredential") +3. Do NOT treat valid alternatives as "advanced" — they're equally valid, just different contexts + +**Example**: Azure Identity SDK has 8 credential types. Pick `DefaultAzureCredential` for SKILL.md (covers most local + production scenarios). Document `ManagedIdentityCredential`, `WorkloadIdentityCredential`, `AzureCliCredential`, etc. in `/references/credential-types.md` without ranking them. + +**Decision rule**: If you're unsure, ask: "Would a user choosing the other approach call what I wrote wrong?" If no, both are core workflows—acknowledge both with trade-offs. + +--- + ### Skill Section Order Follow this structure (based on existing Azure SDK skills): @@ -109,10 +188,10 @@ Follow this structure (based on existing Azure SDK skills): 2. **Installation** — `pip install`, `npm install`, etc. 3. **Environment Variables** — Required configuration, with an inline comment explaining when it's required . If using `DefaultAzureCredential`in production,include `AZURE_TOKEN_CREDENTIALS` (set to `prod` or ``) 4. **Authentication & Lifecycle** — Use a specific Microsoft Entra Token credential like `ManagedIdentityCredential` or `WorkloadIdentityCredential` for production. `DefaultAzureCredential` is only recommended for local development. To use DefaultAzureCredential in production, set the environment variable `AZURE_TOKEN_CREDENTIALS` to `prod` or the specific target credential. **For Python skills, this section MUST start with the standard callout block** (see [Required Authentication & Lifecycle Callout (Python)](#required-authentication--lifecycle-callout-python) below). -5. **Core Workflow** — Minimal viable example +5. **Core Workflow** — Minimal viable example (per core workflow discipline above) 6. **Feature Tables** — Clients, methods, tools 7. **Best Practices** — Numbered list -8. **Reference Links** — Table linking to `/references/*.md` +8. **Reference Links** — Table linking to `/references/*.md` (for Azure SDK skills, include `capabilities.md` + `non-hero-scenarios.md`) ### Required Authentication & Lifecycle Callout (Python) @@ -262,6 +341,97 @@ let client = BlobServiceClient::new( **Never hardcode credentials. Use environment variables.** +### Anti-Patterns: What NOT to Do (REQUIRED Reading) + +**These patterns cause bloat and inefficiency. Every skill author must review this section before writing.** + +#### Anti-Pattern 1: "Exhaustive API Reference" + +- ❌ **Don't**: List all 50 SDK methods in a feature table with code samples for every variant +- ✅ **Do**: Show 3-5 core methods in a table; link to official Azure API reference for exhaustive list +- **Token cost**: Listing all methods + examples = 400-600 tokens wasted +- **User impact**: Overwhelming cognitive load; users don't know what to use + +#### Anti-Pattern 2: "Multiple Ways to Solve One Problem" + +- ❌ **Don't**: "Here's approach A, B, C, and D to paginate results" in the main body +- ✅ **Do**: "Use `ItemPaged` for sync pagination" (primary example); link alternatives to `/references/` +- **Token cost**: Each alternate approach = 50-100 tokens; 5 approaches = skill becomes inefficient +- **User impact**: Decision paralysis; users re-read everything + +#### Anti-Pattern 3: "Beginner + Intermediate + Advanced in One Skill" + +- ❌ **Don't**: Skill that goes from "what is a client?" to "custom retry policies" to "circuit breaker patterns" +- ✅ **Do**: Core workflow covers 80% use case; advanced patterns in `/references/` +- **Token cost**: Every skill level adds 200-300 tokens; three levels = 600-900 extra tokens +- **User impact**: Experts bored, beginners overwhelmed; nobody gets what they need + +#### Anti-Pattern 4: "Restating Official Documentation" + +- ❌ **Don't**: "The CosmosClient constructor takes an endpoint (string) and credential (TokenCredential). The endpoint identifies the Azure Cosmos resource..." +- ✅ **Do**: Show code: `client = CosmosClient(endpoint, credential)`. Link to official docs: `microsoft-docs` MCP. +- **Token cost**: Verbose explanation = 50-100 tokens per parameter; large APIs waste 300+ tokens +- **User impact**: Redundant; official docs are authoritative, skill should show usage not repeat them + +#### Anti-Pattern 5: "Verbose Explanation When Example Suffices" + +- ❌ **Don't**: "To create a client, you first instantiate the class using the constructor, passing the endpoint and credential parameters. The endpoint is a string that identifies your resource..." +- ✅ **Do**: Show code immediately: `with CosmosClient(endpoint, credential) as client:` + +--- + +### Vally Efficiency Validation (REQUIRED - Phase 2) + +**During authoring, validate skill efficiency against vally framework before publication.** + +**1. Measure token count:** + +Use a token counter or model playground to measure each section. Compare to the Token Budget Guidelines targets above. If any section exceeds max, move content to `/references/`. + +**2. Run anti-pattern checklist:** + +- [ ] No exhaustive API reference (show 3-5 core methods, not 50) +- [ ] No multiple solutions to one problem in SKILL.md +- [ ] No beginner+intermediate+advanced mixed +- [ ] No restating official docs (code first, link to microsoft-docs) +- [ ] No verbose prose (examples first, minimal text) + +**3. Example count audit:** + +- [ ] 1-2 complete examples in core workflow (not 5+) +- [ ] Feature table includes 3-5 core methods (not comprehensive API) +- [ ] Max 1 example per best practice bullet + +**4. Frontmatter validation:** + +- [ ] `name` matches `.github/skills//SKILL.md` +- [ ] `description` includes trigger keywords +- [ ] `description` is ≤200 chars +- [ ] Optional `benchmark_tokens` and `benchmark_quality` included + +**4b. Authentication guidance validation** (critical for all credentials): + +- [ ] If skill uses Azure Identity credentials, verify guidance against [official credential-chains docs](https://learn.microsoft.com/en-us/azure/developer/python/sdk/authentication/credential-chains#usage-guidance-for-defaultazurecredential) +- [ ] Development guidance: OK to recommend `DefaultAzureCredential` (supports multiple dev credential types) +- [ ] Production guidance: Must NOT recommend `DefaultAzureCredential` alone; recommend specific credential (e.g., `ManagedIdentityCredential`) +- [ ] Link to `/references/auth-strategies.md` or official docs for production credential selection + +**5. Spot check:** + +- [ ] Can a user copy the core workflow and run it immediately? +- [ ] Do all examples follow best practices (context managers, appropriate credentials)? +- [ ] Are all environment variables documented? + +**Output:** After validation, annotate the skill header with measured token count: + +```markdown +# Azure Service SDK + + +``` + +--- + ### Standard Verb Patterns Azure SDKs use consistent verbs across all languages: @@ -304,15 +474,15 @@ Add both items verbatim (adapted only for language/SDK specifics) as the **first **Variants to apply when the SDK shape differs:** -| Skill type | Adjust item #1 to | Adjust item #2 to | -| ---------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Async-only SDK (e.g. voicelive) | "This SDK is async-only; use the `.aio` namespace throughout." | keep standard | -| Async-first framework (agent framework, m365-agents) | "This SDK is async-first — use `async def` handlers and `async with` throughout." | keep standard | -| Provider-pattern (OpenTelemetry exporters/distro) | keep standard | "Call `provider.shutdown()` / `flush()` at process exit to flush telemetry — providers are not context managers." | -| REST-over-httpx skills | keep standard | "Use `with httpx.Client(...) as client:` (sync) or `async with httpx.AsyncClient(...) as client:` (async) so connections pool and close deterministically." | -| Identity skill | keep standard | "Use credentials as context managers (`with DefaultAzureCredential() as credential:`) when they own token caches / HTTP transports you want cleaned up; for async, use `async with` on credentials from `azure.identity.aio`." | -| FastAPI (non-Azure) | "Pick `def` or `async def` per endpoint based on whether you call async I/O; do not mix sync and blocking calls in one handler." | "Manage long-lived resources (DB pools, HTTP clients) in `lifespan` and inject via `Depends`; use `with`/`async with` for per-request resources." | -| Pure model/schema skill (no I/O, e.g. pydantic) | **skip both** — not applicable | **skip** | +| Skill type | Adjust item #1 to | Adjust item #2 to | +| ----------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Async-only SDK (e.g. voicelive) | "This SDK is async-only; use the `.aio` namespace throughout." | keep standard | +| Framework guidance that is async-oriented (for example some agent frameworks) | "Use the framework's documented async patterns where required, but do not claim async is globally preferred for Azure Python SDKs." | keep standard | +| Provider-pattern (OpenTelemetry exporters/distro) | keep standard | "Call `provider.shutdown()` / `flush()` at process exit to flush telemetry — providers are not context managers." | +| REST-over-httpx skills | keep standard | "Use `with httpx.Client(...) as client:` (sync) or `async with httpx.AsyncClient(...) as client:` (async) so connections pool and close deterministically." | +| Identity skill | keep standard | "Use credentials as context managers (`with DefaultAzureCredential() as credential:`) when they own token caches / HTTP transports you want cleaned up; for async, use `async with` on credentials from `azure.identity.aio`." | +| FastAPI (non-Azure) | "Pick `def` or `async def` per endpoint based on whether you call async I/O; do not mix sync and blocking calls in one handler." | "Manage long-lived resources (DB pools, HTTP clients) in `lifespan` and inject via `Depends`; use `with`/`async with` for per-request resources." | +| Pure model/schema skill (no I/O, e.g. pydantic) | **skip both** — not applicable | **skip** | **Enforcement in code examples.** Every code example inside the skill must itself obey both rules, so the skill demonstrates what it prescribes: @@ -336,6 +506,30 @@ Add both items verbatim (adapted only for language/SDK specifics) as the **first 6. **Always verify package versions using crates.io.** Before using a package, check its version on [crates.io](https://crates.io/) to ensure you are using a stable and supported release. +### Example Effective Skills (Benchmark These) + +**Study these 4 skills for conciseness, focus, and efficiency.** Your new skills should match this profile or explicitly explain why they're larger. + +| Skill | Est. Tokens | Tokens Δ | Quality Δ | Key Pattern to Copy | +| --------------------------- | ----------- | ----------- | ---------- | --------------------------------------------------------- | +| `azure-ai-ml-py` | ~1,200 | **-39%** ✅ | Maintained | Single workflow: train + register model; concise examples | +| `azure-ai-textanalytics-py` | ~1,100 | **-39%** ✅ | Maintained | Feature table (5-7 core methods); minimal prose | +| `azure-ai-contentsafety-py` | ~1,150 | **-37%** ✅ | Maintained | Compact use-case focus; each example is 1-2 lines | +| `azure-appconfiguration-py` | ~1,180 | **-34%** ✅ | Maintained | Strategic link to references; no comprehensive API doc | + +**Why these are effective**: + +1. Stay under 1,200 tokens (core limit) +2. Show ONE canonical workflow, not 5 variants +3. Show 1-2 examples per concept, not 3-5 +4. Use tables for API summary (not prose descriptions) +5. Link to official docs via `microsoft-docs` MCP instead of duplicating +6. Move advanced patterns to `/references/` + +**Before writing your skill**: Ask "Which of these 4 is my use case most similar to?" then mirror that structure. + +--- + ### Handling Deprecated or Rebranded SDKs When an Azure SDK has been deprecated or rebranded, update skills to guide users toward the current package while maintaining backward compatibility: @@ -471,10 +665,12 @@ item = client.create_item(name="example", data={...}) ## Reference Files -| File | Contents | -| -------------------------------------------------- | ------------------------ | -| [references/tools.md](references/tools.md) | Tool integrations | -| [references/streaming.md](references/streaming.md) | Event streaming patterns | +| File | Contents | +| -------------------------------------------------------------------- | ------------------------------------------------------ | +| [references/capabilities.md](references/capabilities.md) | Capability index (hero coverage + links to deep-dives) | +| [references/non-hero-scenarios.md](references/non-hero-scenarios.md) | Concrete non-hero examples | +| [references/tools.md](references/tools.md) | Tool integrations | +| [references/streaming.md](references/streaming.md) | Event streaming patterns | ``` --- @@ -577,17 +773,43 @@ Skills are organized by **language** and **product area** in the `skills/` direc **Write bundled resources first**, then SKILL.md. -**Frontmatter:** +**Quality assurance before finalizing:** + +1. Measure section token counts as you write (use model playground token counter) +2. Compare to Token Budget Guidelines targets +3. Validate against anti-patterns checklist (see Anti-Patterns section) +4. Extract to `/references/` if section exceeds max tokens +5. Run Vally efficiency validation checklist (see Vally Efficiency Validation) +6. Update frontmatter with `benchmark_tokens` and `benchmark_quality` metadata +7. Add token count comment to skill header for future maintenance + +**Frontmatter (Enhanced with Benchmarking Metadata):** ```yaml --- -name: skill-name-py +name: azure-service-py description: | Azure Service SDK for Python. Use for [specific features]. Triggers: "service name", "create resource", "specific operation". +benchmark_tokens: + estimated: 1180 + target: 1100 + max: 1500 +benchmark_quality: + single_core_workflow: true + examples_focused: true + no_prose_bloat: true + anti_patterns_checked: true --- ``` +**Metadata fields:** + +- `benchmark_tokens.estimated` — Actual measured token count +- `benchmark_tokens.target` — Target efficiency (typically 1100) +- `benchmark_tokens.max` — Absolute ceiling (1500; split if exceeded) +- `benchmark_quality.*` — Boolean checklist for validation + ### Step 5: Categorize with Symlinks After creating the skill in `.github/skills/`, create a symlink in the appropriate category: @@ -629,6 +851,8 @@ ls -la skills/python/foundry/agents **Location:** `tests/scenarios//acceptance-criteria.md` +> Keep acceptance criteria in the `tests/` tree (never beside `SKILL.md` inside the skill folder). + **Source materials** (in priority order): 1. Official Microsoft Learn docs (via `microsoft-docs` MCP) @@ -827,7 +1051,36 @@ For Azure SDK language skills, use official upstream source docs and examples as - Update "Best Practices" and "Reference Links" when upstream recommendations change - For Rust, if code uses `azure_core` types/imports directly, ensure `azure_core` is present in `Cargo.toml`; if only service-crate re-exports are used, direct `azure_core` dependency is optional -3. **Validate regenerated skill behavior** +### API Surface Parity Gate (required for every regenerated skill) + +Treat Microsoft Learn API reference as the contract for every snippet in the regenerated skill. + +Before finalizing any regenerated skill: + +1. Identify each SDK type/method shown in snippets (clients, operation groups, model constructors, enum members, long-running methods like `begin_*`). +2. Verify each symbol and signature against current Microsoft Learn API reference pages for that package. +3. If Learn shows a different shape (for example nested `properties=...` models, renamed methods, `begin_*` LRO methods), update the snippet to the Learn shape. +4. Re-check imports so model/client modules match Learn exactly. +5. Do not keep compatibility shortcuts that contradict Learn examples in primary snippets. + +### Scenario Coverage Gate (required for every regenerated skill, all languages) + +Regeneration is not complete when snippets compile — it is complete when the skill demonstrates real usage breadth. + +Before finalizing any regenerated skill: + +1. Identify **hero scenarios** from current Learn quickstarts/tutorials/samples for that SDK. +2. Ensure each hero scenario is represented in the skill with copy-pastable snippets (or an explicit link to a bundled reference file when too large). +3. Add/refresh test scenarios so hero flows are validated by harness patterns. +4. Add at least **one important non-hero scenario** (for example: update/patch, delete/cleanup, export/import, advanced auth mode, paging/filtering, retries/error handling, or LRO monitoring) when supported by the SDK. For Python SDKs that support both sync and async clients, present both forms with equal priority; do not treat either as universally preferred. +5. For Azure SDK skills, structure `references/` as: + - `references/capabilities.md` as a concise index (hero scenarios + links to deeper non-hero references, no historical/migration narration). + - `references/non-hero-scenarios.md` for concrete non-hero examples that are intentionally kept out of the main `SKILL.md`. + - Additional `references/*.md` files for specialized deep-dives (operation groups, tools, evaluator matrices, etc.). +6. If the SDK has broad operation-group coverage (common in management SDKs), include an operation-group table and explicitly call out which groups are covered in snippets vs. referenced only. +7. Never claim "full API surface" unless the skill genuinely demonstrates all major operation groups; otherwise state that the skill is optimized for hero workflows plus selected secondary scenarios. + +8. **Validate regenerated skill behavior** ```bash cd tests @@ -866,6 +1119,66 @@ In the PR/commit notes, include: - Which snippets/signatures were corrected - Which tests/evals were run and their outcomes +#### Python plugin batch recipe: `azure-sdk-python` + +Use this when the request is "regenerate all Python skills under azure-sdk-python." + +1. **Scope the exact targets first** + +```bash +# Canonical source of truth for Python plugin skills +ls .github/plugins/azure-sdk-python/skills/*/SKILL.md +``` + +- Treat `.github/plugins/azure-sdk-python/skills/` as canonical. +- Keep `.github/skills/` links in sync after edits (symlink check/fix step below). + +2. **For each skill, refresh from authoritative sources** + +- Always use `microsoft-docs` MCP first for current Microsoft Learn API guidance. +- Verify installed package API surface with `pip show ` before finalizing snippets. +- For Azure SDK skills, prefer package overview + official SDK repo examples. +- For non-Azure Python skills in this plugin (for example `fastapi-router-py`, `pydantic-models-py`), keep language-specific best-practice variants and skip Azure-specific auth callouts when lifecycle/auth is not applicable. + +3. **Apply Python enforcement rules consistently** + +- Keep the standard section order for Azure SDK Python skills. +- Ensure `## Authentication & Lifecycle` starts with the required callout block (verbatim) when applicable. +- Ensure every client example uses `with` / `async with` lifecycle patterns. +- Ensure `## Best Practices` starts with the two required user-facing rules (or the documented variant for async-only/provider-pattern skills). +- Ensure each regenerated Azure SDK Python skill has `references/capabilities.md` (index) and `references/non-hero-scenarios.md` (concrete non-hero examples). +- Keep existing references/assets/scripts unless stale or incorrect. + +4. **Validate all regenerated Python skills** + +```bash +# Fast frontmatter/structure validation for every Python skill +python .github/skills/skill-creator/scripts/quick_validate.py .github/plugins/azure-sdk-python/skills/ + +# Run Python skill harness in mock mode (all *-py scenarios) +cd tests +pwsh ./run-harness-by-language.ps1 -Language py -Mock +``` + +5. **Sync skill links and docs artifacts** + +```bash +# Ensure .github/skills links point at plugin canonical skills +python .github/scripts/sync_skill_links.py --plugin azure-sdk-python --check +python .github/scripts/sync_skill_links.py --plugin azure-sdk-python --apply + +# Refresh docs site data after content changes +cd docs-site && npx tsx scripts/extract-skills.ts +cd docs-site && npm run build +``` + +6. **Completion criteria for batch regeneration** + +- Every targeted `.github/plugins/azure-sdk-python/skills/*/SKILL.md` is updated or explicitly confirmed current. +- Harness mock run for `-py` skills passes without regressions. +- Skill links are in sync for `azure-sdk-python`. +- PR notes include upstream docs used, signature corrections, and validation outcomes. + --- ## Progressive Disclosure Patterns @@ -938,6 +1251,9 @@ azure-ai-agents/ | Use wrong import paths | Azure SDKs have specific module structures | | Omit sync/async + context-manager bullets from Best Practices in Python skills | End users won't follow rules that aren't written down; examples alone aren't enough | | Mix sync and async in the same Python example | Demonstrates the anti-pattern the skill is supposed to prevent | +| Ship regenerated skills with zero test scenarios | Hero workflows and regressions cannot be validated | +| Claim full API coverage from a single happy-path sample | Hides operation-group and non-hero gaps users need for production | +| Omit `references/*.md` coverage for non-hero capabilities | Forces advanced capabilities out of context and leaves API breadth undocumented | --- @@ -949,6 +1265,7 @@ Before completing a skill: - [ ] User provided SDK package name or documentation URL - [ ] Verified SDK patterns via `microsoft-docs` MCP +- [ ] Verified every snippet's API surface against current Microsoft Learn API reference pages for that SDK **Skill Creation:** @@ -956,7 +1273,12 @@ Before completing a skill: - [ ] SKILL.md under 500 lines - [ ] Authentication follows language rules (`DefaultAzureCredential` for Python/.NET/Java/TS/Go local dev; `DeveloperToolsCredential` local dev + `ManagedIdentityCredential` production for Rust) - [ ] Includes cleanup/delete in examples -- [ ] References organized by feature +- [ ] References organized by feature (`capabilities.md` index + dedicated deep-dive files) +- [ ] Hero scenarios from current Learn docs are explicitly covered in snippets and tests +- [ ] At least one high-value non-hero scenario is included (or explicitly documented as intentionally out of scope) +- [ ] For Azure SDK skills, `references/capabilities.md` indexes hero/non-hero coverage and links to dedicated non-hero docs +- [ ] For Azure SDK skills, `references/non-hero-scenarios.md` contains concrete non-hero examples distinct from hero snippets +- [ ] For broad SDKs (especially management SDKs), operation-group coverage is explicit (covered in snippets vs. reference-only) - [ ] **(Python skills only) Best Practices section contains the two user-facing rules** (sync-or-async consistency + context managers for clients and async credentials), using the variant matched to the skill type - [ ] For Rust skills: `## Best Practices` starts with cargo dependency rule + `azure_core` direct-import rule @@ -970,6 +1292,7 @@ Before completing a skill: - [ ] `tests/scenarios//acceptance-criteria.md` created with correct/incorrect patterns - [ ] `tests/scenarios//scenarios.yaml` created +- [ ] At least one hero scenario and one non-hero scenario are test-covered (when the SDK supports both) - [ ] All scenarios pass (`pnpm harness --mock`) - [ ] Import paths documented precisely diff --git a/.github/skills/skill-creator/references/azure-sdk-patterns.md b/.github/skills/skill-creator/references/azure-sdk-patterns.md index ca7922c7..dc2108d3 100644 --- a/.github/skills/skill-creator/references/azure-sdk-patterns.md +++ b/.github/skills/skill-creator/references/azure-sdk-patterns.md @@ -9,14 +9,15 @@ Reference for creating skills that teach agents to write code following official ## Table of Contents 1. [Core Principles (All Languages)](#core-principles-all-languages) -2. [Standard Naming Conventions](#standard-naming-conventions) -3. [Python Patterns](#python-patterns) -4. [.NET (C#) Patterns](#net-c-patterns) -5. [Java Patterns](#java-patterns) -6. [TypeScript/JavaScript Patterns](#typescriptjavascript-patterns) -7. [Rust Patterns](#rust-patterns) -8. [Authentication (All Languages)](#authentication-all-languages) -9. [Quick Reference Tables](#quick-reference-tables) +2. [Skill Reference Directory Pattern](#skill-reference-directory-pattern) +3. [Standard Naming Conventions](#standard-naming-conventions) +4. [Python Patterns](#python-patterns) +5. [.NET (C#) Patterns](#net-c-patterns) +6. [Java Patterns](#java-patterns) +7. [TypeScript/JavaScript Patterns](#typescriptjavascript-patterns) +8. [Rust Patterns](#rust-patterns) +9. [Authentication (All Languages)](#authentication-all-languages) +10. [Quick Reference Tables](#quick-reference-tables) --- @@ -36,6 +37,20 @@ Azure SDKs follow five design principles. Skills should reinforce these: --- +## Skill Reference Directory Pattern + +For Azure SDK skills, keep `SKILL.md` focused on hero flows and use `references/` for overflow details: + +- `references/capabilities.md` is an index only: hero scenarios covered in `SKILL.md`, non-hero scenario list, and links to deep-dive reference files. +- `references/non-hero-scenarios.md` contains concrete non-hero examples intentionally kept out of `SKILL.md`. +- Additional `references/*.md` files are optional for specialized topics (operation groups, evaluator/tool matrices, migration notes). + +Use present-tense guidance in reference files; avoid historical migration notes in user-facing capability indexes. + +For Python SDK skills that provide both sync and async clients, present both forms as first-class options. Do not encode a blanket preference for either mode in capability prioritization. + +--- + ## Standard Naming Conventions ### Namespace/Package Format diff --git a/.github/workflows/vally-evaluation.yml b/.github/workflows/run-vally-evaluations.yml similarity index 74% rename from .github/workflows/vally-evaluation.yml rename to .github/workflows/run-vally-evaluations.yml index b5975531..bf6f22f7 100644 --- a/.github/workflows/vally-evaluation.yml +++ b/.github/workflows/run-vally-evaluations.yml @@ -1,15 +1,15 @@ -name: Vally Evaluation +name: Vally Workflow Evaluation on: pull_request: paths: - "tests/scenarios/**/vally/**" - - ".github/workflows/vally-evaluation.yml" + - ".github/workflows/run-vally-evaluations.yml" push: branches: [main] paths: - "tests/scenarios/**/vally/**" - - ".github/workflows/vally-evaluation.yml" + - ".github/workflows/run-vally-evaluations.yml" workflow_dispatch: inputs: eval_spec: @@ -19,13 +19,21 @@ on: permissions: contents: read + # eval.yaml's `executor: copilot-sdk` makes vally spawn the real GitHub + # Copilot CLI (`@github/copilot`, pulled in transitively via the vally npm + # dependency tree -- no extra install step needed) to run each stimulus. + # That CLI authenticates via the GH_TOKEN/GITHUB_TOKEN env var, which + # requires the GITHUB_TOKEN to carry the "Copilot Requests" permission. + copilot-requests: write jobs: vally: runs-on: ubuntu-latest env: - GH_TOKEN: ${{ secrets.COPILOT_TOKEN || '' }} - + # Authenticates Copilot CLI and Vally's Copilot SDK evaluator path. + GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} + GITHUB_COPILOT_API_TOKEN: ${{ github.token }} steps: - uses: actions/checkout@v4 with: @@ -33,10 +41,16 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: "20" + node-version: "24" - name: Install Vally CLI - run: npm install -g @microsoft/vally-cli@0.6.0 + run: npm install -g @microsoft/vally-cli@0.7.0 + + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true - name: Verify Vally CLI run: vally --version @@ -86,8 +100,8 @@ jobs: else find tests/scenarios -type f \( -name 'eval.yaml' -o -name 'eval.yml' \) -path '*/vally/*' | sort -u > "$specs_file" fi - elif [ "${{ github.event_name }}" = "pull_request" ]; then - collect_from_changed_dirs "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}" | sort -u > "$specs_file" + elif [ "${{ github.event_name }}" = "pull_request" ] || [ "${{ github.event_name }}" = "pull_request_target" ]; then + collect_from_changed_dirs "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" | sort -u > "$specs_file" else base_sha="${{ github.event.before }}" if [ -z "$base_sha" ] || [ "$base_sha" = '0000000000000000000000000000000000000000' ]; then @@ -137,8 +151,13 @@ jobs: --strict done + - name: Skipping Vally evaluations (fork PR) + if: ${{ steps.specs.outputs.has_specs == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} + run: | + echo "::notice::Skipping Vally evaluations on fork PR from ${{ github.event.pull_request.head.repo.full_name }}. Evaluation runs only on PRs targeting the main repository." + - name: Run Vally evaluations - if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN != '' }} + if: ${{ steps.specs.outputs.has_specs == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} shell: bash run: | set -euo pipefail @@ -156,11 +175,6 @@ jobs: --grader-plugin "$GITHUB_WORKSPACE/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure" \ --junit --output-dir vally-results --workers 2 - - name: Skip eval when token is unavailable - if: ${{ steps.specs.outputs.has_specs == 'true' && env.GH_TOKEN == '' }} - run: | - echo "::warning::COPILOT_TOKEN secret is not available. Skipping vally eval run and keeping lint-only validation." - - name: Upload Vally artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test-harness.yml b/.github/workflows/test-harness.yml index 6dc78f91..1805c2da 100644 --- a/.github/workflows/test-harness.yml +++ b/.github/workflows/test-harness.yml @@ -24,9 +24,10 @@ jobs: cache: "pnpm" cache-dependency-path: tests/pnpm-lock.yaml - - name: Verify Rust skill links are in sync + - name: Verify Skill links are in sync run: | python .github/scripts/sync_skill_links.py --plugin azure-sdk-rust --check + python .github/scripts/sync_skill_links.py --plugin azure-sdk-python --check - name: Install dependencies working-directory: tests diff --git a/tests/.gitignore b/tests/.gitignore index 933d3cab..925d65b4 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -10,6 +10,9 @@ results.md results.json *.report.md *.report.json +vally-experiment-results/ +scenario-results/ +coverage/ # ============================================================================= # Node.js diff --git a/tests/compare-experiment.mjs b/tests/compare-experiment.mjs new file mode 100644 index 00000000..1ca0dce2 --- /dev/null +++ b/tests/compare-experiment.mjs @@ -0,0 +1,301 @@ +#!/usr/bin/env node +/** + * compare-experiment.mjs + * + * Produces a detailed A/B comparison for a Vally experiment directory. + * An experiment directory contains one sub-folder per variant, each with + * results.jsonl and run-summary.jsonl. + * + * Usage: + * node compare-experiment.mjs [--baseline ] [--skill ] + * + * Examples: + * node compare-experiment.mjs Q:\src\skills\vally-experiment-results\2026-06-23T20-24-42-839Z + * node compare-experiment.mjs . --baseline sonnet_baseline --skill sonnet_skill + * + * If --baseline / --skill are omitted the script auto-selects: + * - baseline: first variant alphabetically (or variant containing "baseline") + * - skill: second variant (or variant containing "skill") + * + * Exit codes: 0 = success, 1 = usage/parse error. + */ + +import fs from 'fs'; +import path from 'path'; + +// ── CLI ────────────────────────────────────────────────────────────────────── +const args = process.argv.slice(2); +if (args.length === 0 || args[0] === '--help' || args[0] === '-h') { + console.log('Usage: node compare-experiment.mjs [--baseline ] [--skill ]'); + process.exit(0); +} + +const experimentDir = path.resolve(args[0]); + +let baselineVariant = null; +let skillVariant = null; +for (let i = 1; i < args.length; i++) { + if (args[i] === '--baseline' && args[i + 1]) baselineVariant = args[++i]; + if (args[i] === '--skill' && args[i + 1]) skillVariant = args[++i]; +} + +// ── Discover run directories ───────────────────────────────────────────────── +if (!fs.existsSync(experimentDir)) { + console.error(`Error: directory not found: ${experimentDir}`); + process.exit(1); +} + +const runDirs = fs.readdirSync(experimentDir) + .filter(name => { + const full = path.join(experimentDir, name); + return fs.statSync(full).isDirectory() && + fs.existsSync(path.join(full, 'results.jsonl')) && + fs.existsSync(path.join(full, 'run-summary.jsonl')); + }); + +if (runDirs.length < 2) { + console.error(`Error: need at least 2 run directories with results.jsonl in ${experimentDir}`); + console.error(`Found: ${runDirs.join(', ') || '(none)'}`); + process.exit(1); +} + +// Auto-select if not specified +if (!baselineVariant || !skillVariant) { + const hasBL = runDirs.find(d => d.toLowerCase().includes('baseline')); + const hasSK = runDirs.find(d => d.toLowerCase().includes('skill')); + baselineVariant = baselineVariant ?? (hasBL || runDirs[0]); + skillVariant = skillVariant ?? (hasSK || runDirs.find(d => d !== baselineVariant) || runDirs[1]); +} + +for (const v of [baselineVariant, skillVariant]) { + if (!runDirs.includes(v)) { + console.error(`Error: variant "${v}" not found. Available: ${runDirs.join(', ')}`); + process.exit(1); + } +} + +// ── Load results ───────────────────────────────────────────────────────────── +function loadResults(variantName) { + const dir = path.join(experimentDir, variantName); + // results.jsonl may be multi-line (one record per stimulus); aggregate them + const rows = fs.readFileSync(path.join(dir, 'results.jsonl'), 'utf8') + .split('\n').filter(l => l.trim()) + .map(l => JSON.parse(l)); + const summary = JSON.parse(fs.readFileSync(path.join(dir, 'run-summary.jsonl'), 'utf8').trim()); + return { rows, summary, variantName }; +} + +const bData = loadResults(baselineVariant); +const sData = loadResults(skillVariant); + +// ── Aggregate across stimuli ───────────────────────────────────────────────── +function aggregate(data) { + const { rows, summary, variantName } = data; + let totalScore = 0, passCount = 0, errorCount = 0, skillActivations = 0; + let turns = 0, toolCalls = 0, wallTimeMs = 0, durationMs = 0, llmCalls = 0; + let inputTokens = 0, outputTokens = 0, totalTokens = 0; + let cacheReadTokens = 0, cacheWriteTokens = 0; + + const graderMap = {}; // name -> {bScores, sScores} — built later per-side + + for (const row of rows) { + const m = row.trajectory?.metrics ?? {}; + const tok = m.tokenUsage ?? {}; + const gr = row.gradeResult ?? {}; + + totalScore += gr.score ?? 0; + passCount += gr.passed ? 1 : 0; + errorCount += m.errorCount ?? 0; + skillActivations += m.skillActivationCount ?? 0; + turns += m.turnCount ?? 0; + toolCalls += m.toolCallCount ?? 0; + wallTimeMs += m.wallTimeMs ?? 0; + durationMs += row.durationMs ?? 0; + llmCalls += tok.callCount ?? 0; + inputTokens += tok.inputTokens ?? 0; + outputTokens += tok.outputTokens ?? 0; + totalTokens += tok.totalTokens ?? 0; + cacheReadTokens += tok.cacheReadTokens ?? 0; + cacheWriteTokens += tok.cacheWriteTokens ?? 0; + + // Collect per-grader details (top-level detail list only) + for (const g of (gr.details ?? [])) { + if (!graderMap[g.name]) graderMap[g.name] = { scores: [], passes: [] }; + graderMap[g.name].scores.push(g.score ?? null); + graderMap[g.name].passes.push(g.passed ?? null); + } + } + + const n = rows.length || 1; + return { + variantName, + stimulusCount: rows.length, + experiment: summary.experiment, + evalFile: summary.evalFile, + model: summary.resolvedDefaults?.model ?? '?', + skills: summary.resolvedEnvironment?.skills ?? [], + // quality + avgScore: totalScore / n, + passRate: passCount / n, + errorCount, + skillActivations, + // efficiency + turns, llmCalls, toolCalls, + wallTimeSec: (wallTimeMs / 1000), + durationSec: (durationMs / 1000), + // tokens + inputTokens, outputTokens, totalTokens, + cacheReadTokens, cacheWriteTokens, + netNewInputTokens: inputTokens - cacheReadTokens, + cacheReadRate: inputTokens > 0 ? (cacheReadTokens / inputTokens) : 0, + cacheShareOfTotal: totalTokens > 0 ? (cacheReadTokens / totalTokens) : 0, + cacheWriteRate: inputTokens > 0 ? (cacheWriteTokens / inputTokens) : 0, + cacheWriteShareOfTotal: totalTokens > 0 ? (cacheWriteTokens / totalTokens) : 0, + readToWriteEfficiency: cacheWriteTokens > 0 ? (cacheReadTokens / cacheWriteTokens) : null, + // graders + graderMap, + }; +} + +const bS = aggregate(bData); +const sS = aggregate(sData); + +// ── Helpers ─────────────────────────────────────────────────────────────────── +const W = 44; // label column width + +function fmt(v, decimals = 0) { + if (v == null || v === '') return 'N/A'; + if (typeof v === 'boolean') return v ? 'true' : 'false'; + const n = parseFloat(v); + if (isNaN(n)) return String(v); + return decimals > 0 ? n.toFixed(decimals) : n.toLocaleString(); +} + +function pct(a, b) { + if (!b) return ' N/A'; + const d = (a - b) / b * 100; + const s = (d >= 0 ? '+' : '') + d.toFixed(1) + '%'; + return s.padStart(8); +} + +function row(label, bv, sv, lowerBetter, decimals = 0) { + const bvN = parseFloat(bv); + const svN = parseFloat(sv); + let deltaStr = ' '; + let winnerStr = ''; + if (!isNaN(bvN) && !isNaN(svN)) { + deltaStr = pct(svN, bvN); + if (lowerBetter !== null && bvN !== svN) { + winnerStr = lowerBetter + ? (svN < bvN ? ' ← skill' : ' ← baseline') + : (svN > bvN ? ' ← skill' : ' ← baseline'); + } + } + console.log( + label.slice(0, W).padEnd(W), + fmt(bv, decimals).padStart(14), + fmt(sv, decimals).padStart(14), + deltaStr + winnerStr + ); +} + +function divider(title = '') { + if (title) { + console.log('\n' + title); + console.log('─'.repeat(W + 42)); + } else { + console.log('─'.repeat(W + 42)); + } +} + +// ── Header ──────────────────────────────────────────────────────────────────── +console.log('\n' + '═'.repeat(W + 42)); +console.log('EXPERIMENT COMPARISON'); +console.log('═'.repeat(W + 42)); +console.log(`Experiment : ${bS.experiment}`); +console.log(`Eval file : ${bS.evalFile}`); +console.log(`Model : ${bS.model}`); +console.log(`Directory : ${experimentDir}`); +console.log(`Stimuli : ${bS.stimulusCount}`); +console.log(`Baseline : ${bS.variantName} (skills: ${bS.skills.length === 0 ? 'none' : bS.skills.map(s => path.basename(s)).join(', ')})`); +console.log(`Skill : ${sS.variantName} (skills: ${sS.skills.length === 0 ? 'none' : sS.skills.map(s => path.basename(s)).join(', ')})`); +console.log(''); +console.log('Metric'.padEnd(W), 'Baseline'.padStart(14), 'Skill'.padStart(14), ' Delta'); +divider(); + +// ── Quality ─────────────────────────────────────────────────────────────────── +row('Overall score (0-1)', bS.avgScore, sS.avgScore, false, 4); +row('Pass rate (≥threshold)', bS.passRate, sS.passRate, false, 4); +row('Error count', bS.errorCount, sS.errorCount, true); +row('Skill activations', bS.skillActivations, sS.skillActivations, false); + +// ── Efficiency ──────────────────────────────────────────────────────────────── +divider('EFFICIENCY (lower is better)'); +row('Turns (LLM turns)', bS.turns, sS.turns, true); +row('LLM API calls', bS.llmCalls, sS.llmCalls, true); +row('Tool calls total', bS.toolCalls, sS.toolCalls, true); +row('Wall time (sec)', bS.wallTimeSec, sS.wallTimeSec, true, 1); +row('Total duration (sec)', bS.durationSec, sS.durationSec, true, 1); + +// ── Token cost ──────────────────────────────────────────────────────────────── +divider('TOKEN COST (lower is better)'); +row('Total tokens', bS.totalTokens, sS.totalTokens, true); +row('Input tokens', bS.inputTokens, sS.inputTokens, true); +row('Output tokens', bS.outputTokens, sS.outputTokens, true); +row('Cache read tokens (info)', bS.cacheReadTokens, sS.cacheReadTokens, null); +row('Cache write tokens (info)',bS.cacheWriteTokens, sS.cacheWriteTokens, null); +row('Net new input tokens', bS.netNewInputTokens, sS.netNewInputTokens, true); +row('Cache utilization rate', bS.cacheReadRate, sS.cacheReadRate, false, 4); +row('Cache share of total', bS.cacheShareOfTotal, sS.cacheShareOfTotal, false, 4); +row('Cache write rate', bS.cacheWriteRate, sS.cacheWriteRate, true, 4); +row('Cache write share total', bS.cacheWriteShareOfTotal, sS.cacheWriteShareOfTotal, true, 4); +row('Read/write efficiency', bS.readToWriteEfficiency, sS.readToWriteEfficiency, false, 4); + +// ── Per-grader ──────────────────────────────────────────────────────────────── +divider('PER-GRADER SCORES'); +console.log('Grader'.padEnd(W), 'Baseline'.padStart(14), 'Skill'.padStart(14), ' Delta'); +divider(); + +const allNames = [...new Set([...Object.keys(bS.graderMap), ...Object.keys(sS.graderMap)])]; +for (const name of allNames) { + const bg = bS.graderMap[name]; + const sg = sS.graderMap[name]; + const bScore = bg ? bg.scores.reduce((a, v) => a + (v ?? 0), 0) / bg.scores.length : null; + const sScore = sg ? sg.scores.reduce((a, v) => a + (v ?? 0), 0) / sg.scores.length : null; + const bPass = bg ? (bg.passes.every(p => p) ? 'PASS' : 'FAIL') : '----'; + const sPass = sg ? (sg.passes.every(p => p) ? 'PASS' : 'FAIL') : '----'; + + const bvN = bScore ?? NaN; + const svN = sScore ?? NaN; + let deltaStr = ' ', winnerStr = ''; + if (!isNaN(bvN) && !isNaN(svN)) { + const d = svN - bvN; + deltaStr = ((d >= 0 ? '+' : '') + d.toFixed(3)).padStart(8); + winnerStr = d > 0 ? ' ← skill↑' : d < 0 ? ' ← baseline↑' : ' tie'; + } + + const bCell = `(${bPass}) ${bScore != null ? bScore.toFixed(3) : 'N/A'}`; + const sCell = `(${sPass}) ${sScore != null ? sScore.toFixed(3) : 'N/A'}`; + console.log(name.slice(0, W - 1).padEnd(W), bCell.padStart(14), sCell.padStart(14), deltaStr + winnerStr); +} + +// ── Verdict ─────────────────────────────────────────────────────────────────── +divider(); +const tokenDelta = pct(sS.totalTokens, bS.totalTokens).trim(); +const timeDelta = pct(sS.wallTimeSec, bS.wallTimeSec).trim(); +const scoreDelta = pct(sS.avgScore, bS.avgScore).trim(); +const bothPass = bS.passRate >= 1 && sS.passRate >= 1; + +console.log('\nVERDICT'); +console.log(`Both runs passed threshold : ${bothPass}`); +console.log(`Score delta : ${scoreDelta}`); +console.log(`Token delta : ${tokenDelta}`); +console.log(`Wall-time delta : ${timeDelta}`); +if (bothPass) { + const skillBetter = sS.totalTokens < bS.totalTokens && sS.wallTimeSec < bS.wallTimeSec; + const msg = skillBetter + ? `✅ Skill variant passes with ${tokenDelta} fewer tokens and ${timeDelta} less wall time → SKILL IS BETTER` + : `⚠️ Skill variant does not uniformly reduce both tokens and time — review per-metric deltas above.`; + console.log(msg); +} +console.log(''); diff --git a/tests/generate-skill-experiments.ps1 b/tests/generate-skill-experiments.ps1 new file mode 100644 index 00000000..e2df3ed1 --- /dev/null +++ b/tests/generate-skill-experiments.ps1 @@ -0,0 +1,231 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS +Generate vally skill effectiveness experiments for specified skills. + +.DESCRIPTION +Creates skill_effectiveness_experiment.yaml and skill_effectiveness_eval.yaml files +for all matching skills in the scenarios directory. Each experiment compares +performance with and without the skill context (baseline vs. skill variants). + +Use -SkillPattern to filter by language (e.g., '*-py' for Python, '*-rs' for Rust) +or by service (e.g., 'azure-cosmos*' for Cosmos DB). + +.PARAMETER ScenariosRoot +Root directory containing skill scenario subdirectories. +Defaults to: /scenarios + +.PARAMETER SkillPattern +Filter which skills to process. Supports wildcards. +Defaults to '*-py' (all Python skills). +Examples: -SkillPattern '*-rust', -SkillPattern 'azure-ai-*' + +.PARAMETER DryRun +If set, shows what would be created without actually creating files. + +.PARAMETER Force +If set, overwrites existing experiment files. Otherwise skips existing files. + +.EXAMPLE +./generate-skill-experiments.ps1 +# Generates experiments for all Python skills with default settings + +.EXAMPLE +./generate-skill-experiments.ps1 -SkillPattern '*-rust' +# Generates experiments for all Rust skills + +.EXAMPLE +./generate-skill-experiments.ps1 -SkillPattern 'azure-cosmos*' -DryRun -Verbose +# Preview Cosmos DB experiment files that would be created + +.EXAMPLE +./generate-skill-experiments.ps1 -Force +# Regenerate all experiment files, overwriting existing ones +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ScenariosRoot = (Join-Path $PSScriptRoot "scenarios"), + + [Parameter(Mandatory = $false)] + [string]$SkillPattern = "*-py", + + [Parameter(Mandatory = $false)] + [switch]$DryRun, + + [Parameter(Mandatory = $false)] + [switch]$Force +) + +$ErrorActionPreference = "Stop" + +function New-ExperimentFile { + param( + [Parameter(Mandatory = $true)] + [string]$SkillName, + + [Parameter(Mandatory = $true)] + [string]$SkillPath, + + [Parameter(Mandatory = $true)] + [string]$OutputPath, + + [Parameter(Mandatory = $false)] + [switch]$DryRun + ) + + $experimentName = "$SkillName-skill-experiment" + $content = @" +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: $experimentName +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - $SkillPath + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] +"@ + + if ($DryRun) { + Write-Verbose "Would create: $OutputPath" + Write-Verbose $content + } + else { + Set-Content -Path $OutputPath -Value $content -Encoding UTF8 + Write-Information "✓ Created: $OutputPath" + } +} + +function Copy-EvalFile { + param( + [Parameter(Mandatory = $true)] + [string]$SourcePath, + + [Parameter(Mandatory = $true)] + [string]$DestPath, + + [Parameter(Mandatory = $false)] + [switch]$DryRun + ) + + if (Test-Path $SourcePath) { + if ($DryRun) { + Write-Verbose "Would copy: $SourcePath -> $DestPath" + } + else { + Copy-Item -Path $SourcePath -Destination $DestPath -Force + Write-Information "✓ Copied: $DestPath" + } + return $true + } + return $false +} + +# Find all matching skill scenario directories +$skillDirs = Get-ChildItem -Path $ScenariosRoot -Directory -Filter $SkillPattern | +Sort-Object Name + +Write-Information "Found $($skillDirs.Count) skills matching pattern: $SkillPattern" + +$created = 0 +$skipped = 0 +$failed = 0 + +foreach ($skillDir in $skillDirs) { + $skillName = $skillDir.Name + $vallyDir = Join-Path $skillDir.FullName "vally" + + if (-not (Test-Path $vallyDir)) { + Write-Warning "No vally directory found for $skillName (expected: $vallyDir)" + $skipped++ + continue + } + + # Determine skill path for the skills array + # Extract language suffix from skill name (e.g., azure-cosmos-db-py -> py) + $languageMap = @{ + '-py' = 'azure-sdk-python' + '-dotnet' = 'azure-sdk-dotnet' + '-ts' = 'azure-sdk-typescript' + '-java' = 'azure-sdk-java' + '-rust' = 'azure-sdk-rust' + } + + $language = 'py' # default + foreach ($suffix in $languageMap.Keys) { + if ($skillName -match "$suffix`$") { + $language = $languageMap[$suffix] + break + } + } + + $skillPath = "../../../../.github/plugins/$language/skills/$skillName" + + # Paths for experiment files + $experimentFile = Join-Path $vallyDir "skill_effectiveness_experiment.yaml" + $evalFile = Join-Path $vallyDir "skill_effectiveness_eval.yaml" + $sourceEvalFile = Join-Path $vallyDir "eval.yaml" + + # Create experiment file + if ((Test-Path $experimentFile) -and -not $Force) { + Write-Verbose "Skipping (exists): $experimentFile" + $skipped++ + } + else { + try { + New-ExperimentFile -SkillName $skillName -SkillPath $skillPath -OutputPath $experimentFile -DryRun:$DryRun + $created++ + } + catch { + Write-Error "Failed to create experiment file for $skillName : $_" + $failed++ + } + } + + # Create eval file (copy from eval.yaml) + if ((Test-Path $evalFile) -and -not $Force) { + Write-Verbose "Skipping (exists): $evalFile" + } + else { + try { + if (Copy-EvalFile -SourcePath $sourceEvalFile -DestPath $evalFile -DryRun:$DryRun) { + if (-not $DryRun) { + $created++ + } + } + else { + Write-Warning "Source eval.yaml not found for $skillName" + } + } + catch { + Write-Error "Failed to copy eval file for $skillName : $_" + $failed++ + } + } +} + +Write-Information "" +Write-Information "Summary:" +Write-Information "--------" +Write-Information "Created: $created" +Write-Information "Skipped: $skipped" +if ($failed -gt 0) { + Write-Information "Failed: $failed" +} +if ($DryRun) { + Write-Information "" + Write-Information "DRY RUN - No files were actually created. Use -Force to create files." +} diff --git a/tests/harness/copilot-client.test.ts b/tests/harness/copilot-client.test.ts new file mode 100644 index 00000000..53100cdc --- /dev/null +++ b/tests/harness/copilot-client.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, it } from "vitest"; + +import { + CopilotGenerationError, + SkillCopilotClient, + classifyCopilotError, +} from "./copilot-client.js"; + +describe("SkillCopilotClient.extractCode", () => { + it("keeps assignment line for multiline constructor call without fences", () => { + const client = new SkillCopilotClient(process.cwd(), true); + + const response = [ + "from azure.identity import DefaultAzureCredential", + "from azure.monitor.opentelemetry.exporter import AzureMonitorTraceExporter", + "", + "exporter = AzureMonitorTraceExporter(", + " credential=DefaultAzureCredential(),", + ' storage_directory="/path/to/storage",', + " disable_offline_storage=False,", + ")", + ].join("\n"); + + const extracted = ( + client as unknown as { extractCode: (r: string) => string } + ).extractCode(response); + + expect(extracted).toContain("exporter = AzureMonitorTraceExporter("); + expect(extracted).toContain("disable_offline_storage=False,"); + expect(extracted.trim().endsWith(")")).toBe(true); + }); + + it("prefers fenced code blocks when present", () => { + const client = new SkillCopilotClient(process.cwd(), true); + + const response = [ + "Here is the implementation:", + "```python", + "x = 1", + "print(x)", + "```", + ].join("\n"); + + const extracted = ( + client as unknown as { extractCode: (r: string) => string } + ).extractCode(response); + + expect(extracted).toBe("x = 1\nprint(x)"); + }); +}); + +describe("classifyCopilotError", () => { + it("classifies timeout and marks retryable", () => { + const classified = classifyCopilotError( + new Error("Timeout after 120000ms waiting for session.idle"), + ); + + expect(classified.kind).toBe("timeout"); + expect(classified.retryable).toBe(true); + }); + + it("classifies auth and marks non-retryable", () => { + const classified = classifyCopilotError( + new Error( + "Authentication failed: Failed to fetch GitHub CLI user login (401): Bad credentials", + ), + ); + + expect(classified.kind).toBe("auth"); + expect(classified.retryable).toBe(false); + }); + + it("returns existing classified errors unchanged", () => { + const original = new CopilotGenerationError("transient", "429", true); + const classified = classifyCopilotError(original); + + expect(classified).toBe(original); + }); +}); diff --git a/tests/harness/copilot-client.ts b/tests/harness/copilot-client.ts index 1e05bb6a..9f9cdb50 100644 --- a/tests/harness/copilot-client.ts +++ b/tests/harness/copilot-client.ts @@ -16,6 +16,73 @@ import type { import { DEFAULT_GENERATION_CONFIG } from "./types.js"; import { CopilotClient as SDKCopilotClient } from "@github/copilot-sdk"; +export type CopilotGenerationErrorKind = + | "timeout" + | "auth" + | "transient" + | "fatal"; + +export class CopilotGenerationError extends Error { + readonly kind: CopilotGenerationErrorKind; + readonly retryable: boolean; + + constructor( + kind: CopilotGenerationErrorKind, + message: string, + retryable: boolean, + ) { + super(message); + this.name = "CopilotGenerationError"; + this.kind = kind; + this.retryable = retryable; + } +} + +export function classifyCopilotError(error: unknown): CopilotGenerationError { + if (error instanceof CopilotGenerationError) { + return error; + } + + const message = error instanceof Error ? error.message : String(error); + const normalized = message.toLowerCase(); + + if ( + normalized.includes("authentication failed") || + normalized.includes("bad credentials") || + normalized.includes("failed to fetch github cli user login") || + normalized.includes("401") + ) { + return new CopilotGenerationError("auth", message, false); + } + + if ( + normalized.includes("timeout") || + normalized.includes("session.idle") || + normalized.includes("timed out") + ) { + return new CopilotGenerationError("timeout", message, true); + } + + if ( + normalized.includes("429") || + normalized.includes("rate limit") || + normalized.includes("econnreset") || + normalized.includes("etimedout") || + normalized.includes("enotfound") || + normalized.includes("socket hang up") || + normalized.includes("temporarily unavailable") || + normalized.includes("service unavailable") + ) { + return new CopilotGenerationError("transient", message, true); + } + + return new CopilotGenerationError("fatal", message, false); +} + +function sleep(ms: number): Promise { + return new Promise((resolveSleep) => setTimeout(resolveSleep, ms)); +} + // ============================================================================= // Mock Client // ============================================================================= @@ -85,6 +152,8 @@ export class MockCopilotClient implements CopilotClient { export class SkillCopilotClient implements CopilotClient { private static readonly SKILLS_DIR = ".github/skills"; private static readonly PLUGINS_DIR = ".github/plugins"; + private static readonly DEFAULT_TIMEOUT_MS = 120000; + private static readonly DEFAULT_MAX_RETRIES = 2; private basePath: string; private skillsDir: string; @@ -227,23 +296,28 @@ Generate only code. Follow the patterns from the skill documentation exactly. ): Promise { const startTime = Date.now(); const fullPrompt = this.buildPrompt(prompt, skillContext); + const timeoutMs = this.getTimeoutMs(); + const maxRetries = this.getMaxRetries(); - const client = new SDKCopilotClient(); - - let rawResponse = ""; - - try { - const session = await client.createSession({ - model: config.model, - }); + for (let attempt = 0; attempt <= maxRetries; attempt++) { + const client = new SDKCopilotClient(); + let session: unknown; try { - const response = await session.sendAndWait( - { prompt: fullPrompt }, - 120000, - ); - - rawResponse = response?.data?.content ?? ""; + session = await client.createSession({ + model: config.model, + }); + + const response = await ( + session as { + sendAndWait: ( + request: { prompt: string }, + timeout: number, + ) => Promise<{ data?: { content?: string } }>; + } + ).sendAndWait({ prompt: fullPrompt }, timeoutMs); + + const rawResponse = response?.data?.content ?? ""; const code = this.extractCode(rawResponse); return { @@ -255,20 +329,51 @@ Generate only code. Follow the patterns from the skill documentation exactly. durationMs: Date.now() - startTime, rawResponse, }; + } catch (error) { + const classified = classifyCopilotError(error); + if (!classified.retryable || attempt === maxRetries) { + throw classified; + } + + // Exponential backoff with jitter for transient SDK/network failures. + const backoffMs = Math.min(1000 * 2 ** attempt, 8000); + const jitterMs = Math.floor(Math.random() * 300); + await sleep(backoffMs + jitterMs); } finally { - const s = session as unknown as { + const s = session as { disconnect?: () => Promise; destroy?: () => Promise; }; - if (typeof s.disconnect === "function") { + if (s && typeof s.disconnect === "function") { await s.disconnect(); - } else if (typeof s.destroy === "function") { + } else if (s && typeof s.destroy === "function") { await s.destroy(); } + await client.stop(); } - } finally { - await client.stop(); } + + throw new CopilotGenerationError( + "fatal", + "Code generation failed after all retry attempts.", + false, + ); + } + + private getTimeoutMs(): number { + const raw = process.env["HARNESS_COPILOT_TIMEOUT_MS"]; + const parsed = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isFinite(parsed) && parsed > 0 + ? parsed + : SkillCopilotClient.DEFAULT_TIMEOUT_MS; + } + + private getMaxRetries(): number { + const raw = process.env["HARNESS_COPILOT_MAX_RETRIES"]; + const parsed = raw ? Number.parseInt(raw, 10) : NaN; + return Number.isFinite(parsed) && parsed >= 0 + ? parsed + : SkillCopilotClient.DEFAULT_MAX_RETRIES; } /** @@ -295,9 +400,12 @@ Generate only code. Follow the patterns from the skill documentation exactly. const codeLines: string[] = []; let inCode = false; + const assignmentRegex = /^[A-Za-z_][A-Za-z0-9_]*\s*=\s*.+$/; + const callStartRegex = /^[A-Za-z_][A-Za-z0-9_.]*\s*\(.+$/; + for (const line of lines) { - // Heuristic: lines starting with import, def, class, or indented - if ( + const trimmed = line.trim(); + const isCodeLikeLine = line.startsWith("import ") || line.startsWith("from ") || line.startsWith("def ") || @@ -307,8 +415,12 @@ Generate only code. Follow the patterns from the skill documentation exactly. line.startsWith("let ") || line.startsWith("using ") || line.startsWith(" ") || - line.startsWith("\t") - ) { + line.startsWith("\t") || + assignmentRegex.test(trimmed) || + callStartRegex.test(trimmed); + + // Heuristic: lines starting with import, def, class, or indented + if (isCodeLikeLine) { inCode = true; codeLines.push(line); } else if (inCode && line.trim() === "") { diff --git a/tests/harness/ralph-loop.test.ts b/tests/harness/ralph-loop.test.ts index c322be6e..019c1b42 100644 --- a/tests/harness/ralph-loop.test.ts +++ b/tests/harness/ralph-loop.test.ts @@ -55,10 +55,11 @@ class MockClient implements CopilotClient { _prompt: string, _skillName: string, _config?: GenerationConfig, - _scenarioName?: string + _scenarioName?: string, ): Promise { this.callCount++; - const code = this.responses.get(this.callCount) ?? "# Default mock response\npass"; + const code = + this.responses.get(this.callCount) ?? "# Default mock response\npass"; return { code, prompt: _prompt, @@ -111,13 +112,15 @@ class MockEvaluator { skillName: "test-skill", scenario, generatedCode: code, + rawResponse: code, findings, matchedCorrect: [], matchedIncorrect: score < 70 ? ["some-section"] : [], score, passed: score >= 50, errorCount: findings.filter((f) => f.severity === Severity.ERROR).length, - warningCount: findings.filter((f) => f.severity === Severity.WARNING).length, + warningCount: findings.filter((f) => f.severity === Severity.WARNING) + .length, }; } } @@ -161,7 +164,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80 } + { qualityThreshold: 80 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -184,7 +187,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 10 } + { qualityThreshold: 80, improvementThreshold: 10 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -197,18 +200,14 @@ describe("RalphLoopController", () => { }); it("should stop at max iterations if threshold not met", async () => { - mockClient.setResponses([ - "# Attempt 1", - "# Attempt 2", - "# Attempt 3", - ]); + mockClient.setResponses(["# Attempt 1", "# Attempt 2", "# Attempt 3"]); mockEvaluator.setScores([30, 40, 50]); // Never reaches 80 const controller = new RalphLoopController( criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 5 } + { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -229,7 +228,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { earlyStopOnPerfect: true } + { earlyStopOnPerfect: true }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -247,7 +246,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 5 } + { qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -265,7 +264,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 5 } + { qualityThreshold: 80, improvementThreshold: 5 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -285,7 +284,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 0 } + { maxIterations: 3, qualityThreshold: 80, improvementThreshold: 0 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -305,7 +304,7 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, improvementThreshold: 10 } + { qualityThreshold: 80, improvementThreshold: 10 }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -334,7 +333,11 @@ describe("RalphLoopController", () => { criteria, mockEvaluator as unknown as CodeEvaluator, mockClient, - { qualityThreshold: 80, includeFeedback: true, improvementThreshold: 10 } + { + qualityThreshold: 80, + includeFeedback: true, + improvementThreshold: 10, + }, ); const result = await controller.run("Generate code", "test-scenario"); @@ -353,7 +356,7 @@ describe("RalphLoopController", () => { const controller = new RalphLoopController( criteria, mockEvaluator as unknown as CodeEvaluator, - mockClient + mockClient, ); const result = await controller.run("Generate code", "test-scenario"); diff --git a/tests/harness/runner.ts b/tests/harness/runner.ts index f106cea9..704ac972 100644 --- a/tests/harness/runner.ts +++ b/tests/harness/runner.ts @@ -20,8 +20,17 @@ import type { GenerationConfig, Finding, } from "./types.js"; -import { DEFAULT_GENERATION_CONFIG, Severity, createFinding } from "./types.js"; -import { SkillCopilotClient, checkCopilotAvailable } from "./copilot-client.js"; +import { + DEFAULT_GENERATION_CONFIG, + Severity, + createEvaluationResult, + createFinding, +} from "./types.js"; +import { + CopilotGenerationError, + SkillCopilotClient, + checkCopilotAvailable, +} from "./copilot-client.js"; import { AcceptanceCriteriaLoader } from "./criteria-loader.js"; import { CodeEvaluator } from "./evaluator.js"; import { @@ -282,19 +291,41 @@ export class SkillEvaluationRunner { ); } - // Generate code - const genResult = await this.copilotClient.generate( - scenario.prompt, - skillName, - suite.config, - scenario.name, - ); + let evalResult: EvaluationResult; - // Evaluate - const evalResult = evaluator.evaluate(genResult.code, scenario.name); + try { + // Generate code + const genResult = await this.copilotClient.generate( + scenario.prompt, + skillName, + suite.config, + scenario.name, + ); - // Add scenario-specific checks - this.checkScenarioPatterns(evalResult, scenario, genResult.code); + // Evaluate + evalResult = evaluator.evaluate(genResult.code, scenario.name); + evalResult.rawResponse = genResult.rawResponse; + + // Add scenario-specific checks + this.checkScenarioPatterns(evalResult, scenario, genResult.code); + } catch (error) { + const kind = + error instanceof CopilotGenerationError ? error.kind : "fatal"; + const message = error instanceof Error ? error.message : String(error); + + evalResult = createEvaluationResult(skillName, scenario.name, ""); + evalResult.passed = false; + evalResult.errorCount = 1; + evalResult.score = 0; + evalResult.findings.push( + createFinding({ + severity: Severity.ERROR, + rule: `runtime:${kind}`, + message: `Code generation failed (${kind}): ${message}`, + suggestion: this.getGenerationFailureSuggestion(kind), + }), + ); + } results.push(evalResult); @@ -394,6 +425,21 @@ export class SkillEvaluationRunner { } } + private getGenerationFailureSuggestion( + kind: "timeout" | "auth" | "transient" | "fatal", + ): string { + switch (kind) { + case "timeout": + return "Retry with HARNESS_COPILOT_TIMEOUT_MS set higher (for example 240000), or re-run this scenario."; + case "auth": + return "Refresh GitHub authentication (gh auth login) or set GH_TOKEN/GITHUB_TOKEN with valid Copilot access."; + case "transient": + return "This looks transient; retry the run. If it repeats, verify network stability and API availability."; + default: + return "Inspect diagnostics and rerun with --verbose for additional context."; + } + } + private printFinding(finding: Finding): void { const severityStyle = this.getSeverityStyle(finding.severity); const severityLabel = finding.severity.toUpperCase(); @@ -621,6 +667,7 @@ function summaryToDict(summary: EvaluationSummary): Record { skill_name: r.skillName, scenario: r.scenario, generated_code: r.generatedCode, + raw_response: r.rawResponse, passed: r.passed, score: r.score, error_count: r.errorCount, @@ -770,6 +817,7 @@ function convertRalphToSummary(ralph: RalphLoopSummary): EvaluationSummary { skillName: ralph.skillName, scenario: scenarioName, generatedCode: lastIteration.generatedCode, + rawResponse: "", findings: lastIteration.findings, matchedCorrect: [], matchedIncorrect: [], @@ -813,6 +861,8 @@ interface CLIOptions { threshold?: number; } +type CliErrorKind = "timeout" | "auth" | "transient" | "fatal"; + interface AllSkillsSummary { totalSkills: number; passedSkills: number; @@ -826,6 +876,52 @@ interface AllSkillsSummary { skills: EvaluationSummary[]; } +function classifyCliError(error: unknown): CliErrorKind { + if (error instanceof CopilotGenerationError) { + return error.kind; + } + + const message = error instanceof Error ? error.message.toLowerCase() : ""; + if ( + message.includes("authentication failed") || + message.includes("bad credentials") || + message.includes("401") + ) { + return "auth"; + } + if (message.includes("timeout") || message.includes("session.idle")) { + return "timeout"; + } + if (message.includes("rate limit") || message.includes("429")) { + return "transient"; + } + return "fatal"; +} + +function writeCliErrorOutput( + options: CLIOptions, + skillName: string | undefined, + error: unknown, +): void { + const message = error instanceof Error ? error.message : String(error); + const payload = { + status: "ERROR", + error_kind: classifyCliError(error), + error: message, + skill: skillName ?? null, + timestamp: new Date().toISOString(), + }; + const output = JSON.stringify(payload, null, 2); + + if (options.outputFile) { + writeFileSync(options.outputFile, output); + console.log(`Results written to: ${options.outputFile}`); + return; + } + + console.log(output); +} + async function main(): Promise { const program = new Command(); @@ -1087,6 +1183,10 @@ async function main(): Promise { } } catch (err) { const message = err instanceof Error ? err.message : String(err); + if (options.output === "json") { + writeCliErrorOutput(options, skillArg, err); + return 1; + } console.log(chalk.red(`Error: ${message}`)); return 1; } diff --git a/tests/harness/types.ts b/tests/harness/types.ts index 715fb494..37e5b525 100644 --- a/tests/harness/types.ts +++ b/tests/harness/types.ts @@ -83,6 +83,7 @@ export interface EvaluationResult { skillName: string; scenario: string; generatedCode: string; + rawResponse: string; findings: Finding[]; matchedCorrect: string[]; matchedIncorrect: string[]; @@ -110,7 +111,7 @@ export interface GenerationConfig { * Default generation configuration. */ export const DEFAULT_GENERATION_CONFIG: GenerationConfig = { - model: "gpt-4", + model: "gpt-5.5", maxTokens: 2000, temperature: 0.3, includeSkillContext: true, @@ -175,7 +176,7 @@ export interface CopilotClient { prompt: string, skillName: string, config?: GenerationConfig, - scenarioName?: string + scenarioName?: string, ): Promise; } @@ -223,12 +224,13 @@ export function detectLanguage(skillName: string): Language { export function createEvaluationResult( skillName: string, scenario: string, - generatedCode: string + generatedCode: string, ): EvaluationResult { return { skillName, scenario, generatedCode, + rawResponse: "", findings: [], matchedCorrect: [], matchedIncorrect: [], @@ -242,7 +244,9 @@ export function createEvaluationResult( /** * Create an empty code pattern. */ -export function createCodePattern(partial: Partial = {}): CodePattern { +export function createCodePattern( + partial: Partial = {}, +): CodePattern { return { code: "", language: "python", @@ -256,7 +260,9 @@ export function createCodePattern(partial: Partial = {}): CodePatte /** * Create an empty validation rule. */ -export function createValidationRule(partial: Partial = {}): ValidationRule { +export function createValidationRule( + partial: Partial = {}, +): ValidationRule { return { name: "", description: "", @@ -274,7 +280,7 @@ export function createValidationRule(partial: Partial = {}): Val * Create an empty acceptance criteria. */ export function createAcceptanceCriteria( - partial: Partial = {} + partial: Partial = {}, ): AcceptanceCriteria { return { skillName: "", diff --git a/tests/package-lock.json b/tests/package-lock.json index 1c56a6b4..ee981860 100644 --- a/tests/package-lock.json +++ b/tests/package-lock.json @@ -8,23 +8,120 @@ "name": "agent-skills-tests", "version": "1.0.0", "dependencies": { - "@github/copilot-sdk": "^0.1.20", + "@github/copilot-sdk": "^1.0.3", "chalk": "^5.3.0", "commander": "^12.1.0", "yaml": "^2.4.5", "zod": "^3.23.8" }, "devDependencies": { + "@microsoft/vally": "^0.7.0", "@types/node": "^20.14.0", + "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.5.0", "tsx": "^4.15.6", "typescript": "^5.4.5", - "vitest": "^1.6.0" + "vite": "^8.1.0", + "vitest": "^4.1.9" }, "engines": { "node": ">=20.0.0" } }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@emnapi/core": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", + "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", + "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.3.tgz", @@ -612,26 +709,31 @@ } }, "node_modules/@github/copilot": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-0.0.411.tgz", - "integrity": "sha512-I3/7gw40Iu1O+kTyNPKJHNqDRyOebjsUW6wJsvSVrOpT0TNa3/lfm8xdS2XUuJWkp+PgEG/PRwF7u3DVNdP7bQ==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.65.tgz", + "integrity": "sha512-J1XvLuOiVpiAi/E1MBICBymszCgdGLnZxokosXzGcmcjEVZd+QSDoW/kPRHq6oEyBT9SDASPcjCEZ9Q0rLJllg==", "license": "SEE LICENSE IN LICENSE.md", + "dependencies": { + "detect-libc": "^2.1.2" + }, "bin": { "copilot": "npm-loader.js" }, "optionalDependencies": { - "@github/copilot-darwin-arm64": "0.0.411", - "@github/copilot-darwin-x64": "0.0.411", - "@github/copilot-linux-arm64": "0.0.411", - "@github/copilot-linux-x64": "0.0.411", - "@github/copilot-win32-arm64": "0.0.411", - "@github/copilot-win32-x64": "0.0.411" + "@github/copilot-darwin-arm64": "1.0.65", + "@github/copilot-darwin-x64": "1.0.65", + "@github/copilot-linux-arm64": "1.0.65", + "@github/copilot-linux-x64": "1.0.65", + "@github/copilot-linuxmusl-arm64": "1.0.65", + "@github/copilot-linuxmusl-x64": "1.0.65", + "@github/copilot-win32-arm64": "1.0.65", + "@github/copilot-win32-x64": "1.0.65" } }, "node_modules/@github/copilot-darwin-arm64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-0.0.411.tgz", - "integrity": "sha512-dtr+iHxTS4f8HlV2JT9Fp0FFoxuiPWCnU3XGmrHK+rY6bX5okPC2daU5idvs77WKUGcH8yHTZtfbKYUiMxKosw==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-arm64/-/copilot-darwin-arm64-1.0.65.tgz", + "integrity": "sha512-NFc4xIstZNiIuAYkurQT5DVtbZjBoZ/z6yt/Ffcom7Y5QGjfpN4BFuekv9k+OADRioxxR99NgmhjbuNPWtQhNQ==", "cpu": [ "arm64" ], @@ -645,9 +747,9 @@ } }, "node_modules/@github/copilot-darwin-x64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-0.0.411.tgz", - "integrity": "sha512-zhdbQCbPi1L4iHClackSLx8POfklA+NX9RQLuS48HlKi/0KI/JlaDA/bdbIeMR79wjif5t9gnc/m+RTVmHlRtA==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-darwin-x64/-/copilot-darwin-x64-1.0.65.tgz", + "integrity": "sha512-0wtV22KmTa12VbqWRRkgvJcBz/oIbszfcIpyDWGc4MzbCVksajQ3TWVQ6c7Sdzj5RifCaYdkHAX2zuIYXYlLoQ==", "cpu": [ "x64" ], @@ -661,12 +763,15 @@ } }, "node_modules/@github/copilot-linux-arm64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-0.0.411.tgz", - "integrity": "sha512-oZYZ7oX/7O+jzdTUcHkfD1A8YnNRW6mlUgdPjUg+5rXC43bwIdyatAnc0ObY21m9h8ghxGqholoLhm5WnGv1LQ==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-arm64/-/copilot-linux-arm64-1.0.65.tgz", + "integrity": "sha512-dOwdy/YbTXQN/+x2v4ZgiDycdRtWElyHxPuA6ail3yJDt0nagwn8OYAA/diBLPMAJuuBXiOZGvvb9fGRuh7Xgg==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -677,12 +782,15 @@ } }, "node_modules/@github/copilot-linux-x64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-0.0.411.tgz", - "integrity": "sha512-nnXrKANmmGnkwa3ROlKdAhVNOx8daeMSE8Xh0o3ybKckFv4s38blhKdcxs0RJQRxgAk4p7XXGlDDKNRhurqF1g==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-linux-x64/-/copilot-linux-x64-1.0.65.tgz", + "integrity": "sha512-al/1a/l/GrpHtygTxt7PZspmv0eHBPdAZ5B31J7Hv/GRdVZM4STCC9dCIOSUFsOX2fhaKD8yLfz4HureSYs23g==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -692,18 +800,56 @@ "copilot-linux-x64": "copilot" } }, + "node_modules/@github/copilot-linuxmusl-arm64": { + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-arm64/-/copilot-linuxmusl-arm64-1.0.65.tgz", + "integrity": "sha512-xccQeJSR45xyoaL7J5mZjtU++dmte+ZCDQkIlrpTn2yuMl2LWriBvorQ1P2MwVnXmIiW/GHi93B+lNtsybA9yw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-arm64": "copilot" + } + }, + "node_modules/@github/copilot-linuxmusl-x64": { + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-linuxmusl-x64/-/copilot-linuxmusl-x64-1.0.65.tgz", + "integrity": "sha512-RHPVUaqjSrhKHQ2EpfGKWErnV+R5elGIZaHXPKO10zpSaQD9b/C9u6nLigZnBuT/8sCaJpVrazPMwOYvYA62aw==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "SEE LICENSE IN LICENSE.md", + "optional": true, + "os": [ + "linux" + ], + "bin": { + "copilot-linuxmusl-x64": "copilot" + } + }, "node_modules/@github/copilot-sdk": { - "version": "0.1.25", - "resolved": "https://registry.npmjs.org/@github/copilot-sdk/-/copilot-sdk-0.1.25.tgz", - "integrity": "sha512-hIgYLPXzWw9bNgrsD5BLKmgVH20ow5Or5UyVXfVe3YgeiaTgFxC4jWSAVHLGB6ufHZUrvbjppcq2dWK63FmDRA==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@github/copilot-sdk/-/copilot-sdk-1.0.4.tgz", + "integrity": "sha512-c6/gpatP7LAuP9stlc2uXXOrB+TJMFSs3Jrzu9FG8lJJYFGmdzbBvGz2UdL1jww84fp6D7cohEZa4UGq1+iuyA==", "license": "MIT", "dependencies": { - "@github/copilot": "^0.0.411", + "@github/copilot": "^1.0.65", "vscode-jsonrpc": "^8.2.1", "zod": "^4.3.6" }, "engines": { - "node": ">=20.0.0" + "node": "^20.19.0 || >=22.12.0" } }, "node_modules/@github/copilot-sdk/node_modules/zod": { @@ -716,9 +862,9 @@ } }, "node_modules/@github/copilot-win32-arm64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-0.0.411.tgz", - "integrity": "sha512-h+Bovb2YVCQSeELZOO7zxv8uht45XHcvAkFbRsc1gf9dl109sSUJIcB4KAhs8Aznk28qksxz7kvdSgUWyQBlIA==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-arm64/-/copilot-win32-arm64-1.0.65.tgz", + "integrity": "sha512-/vSE/t9Wm3eFSWpxlKyn/oL8OAVOB0yFO7ECxhgbtiqNrBd1tgpYh1k7IXBIWa/saxlV1+de6DEmPuQfx3Z0bg==", "cpu": [ "arm64" ], @@ -732,9 +878,9 @@ } }, "node_modules/@github/copilot-win32-x64": { - "version": "0.0.411", - "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-0.0.411.tgz", - "integrity": "sha512-xmOgi1lGvUBHQJWmq5AK1EP95+Y8xR4TFoK9OCSOaGbQ+LFcX2jF7iavnMolfWwddabew/AMQjsEHlXvbgMG8Q==", + "version": "1.0.65", + "resolved": "https://registry.npmjs.org/@github/copilot-win32-x64/-/copilot-win32-x64-1.0.65.tgz", + "integrity": "sha512-wjVWXepET+SpFg8z8V43ZiTy6X1OerCb7yu3QZKNNJ3zY9z20goihPXQCDWkiJpGzszNSgfrsiqUzpUsC9qS0A==", "cpu": [ "x64" ], @@ -799,17 +945,14 @@ "url": "https://github.com/sponsors/nzakas" } }, - "node_modules/@jest/schemas": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", - "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", "dev": true, "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.27.8" - }, "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + "node": ">=6.0.0" } }, "node_modules/@jridgewell/sourcemap-codec": { @@ -819,24 +962,84 @@ "dev": true, "license": "MIT" }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", - "integrity": "sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==", - "cpu": [ - "arm" - ], + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@microsoft/vally": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@microsoft/vally/-/vally-0.7.0.tgz", + "integrity": "sha512-GYxSgub10SC6X1d/vVUCOorGzAMWIUN7uk2zq65pPle0CfKdWy+HiEpnJkgrXGat7kZxzx+jM9lrffW3nfKpmw==", + "dev": true, + "dependencies": { + "@github/copilot-sdk": "^1.0.3", + "@opentelemetry/api": "^1.9.1", + "js-tiktoken": "^1.0.21", + "picomatch": "^4.0.4", + "yaml": "^2.9.0", + "zod": "^4.4.3" + } + }, + "node_modules/@microsoft/vally/node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", + "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", "dev": true, "license": "MIT", "optional": true, - "os": [ - "android" - ] + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.57.1.tgz", - "integrity": "sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==", + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.137.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.137.0.tgz", + "integrity": "sha512-WT+Gb24i8hmvo85AIv2oEYouEXkRlKAlT9WaCa3TfLgNCN+GhrJOGZuIlMouAh38Qe4QOx26eUOVsq70qXrywA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.3.tgz", + "integrity": "sha512-DT6Z3PhvioeHMvxo+xHc3KtqggrI7CCTXCmC2h/5zUlp5jVitv7XEy+9q5/7v8IolhlioawpMo8Kg0EEBy7J0g==", "cpu": [ "arm64" ], @@ -845,12 +1048,15 @@ "optional": true, "os": [ "android" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.57.1.tgz", - "integrity": "sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==", + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.3.tgz", + "integrity": "sha512-0NwgwsjM7LrsuVnXMK3koTpagBNOhloc/BNjKqZjv4V5zI5r13qx69uVhRx+o5Z0yy4Hzq+lpy7TAgUG/ocvrw==", "cpu": [ "arm64" ], @@ -859,12 +1065,15 @@ "optional": true, "os": [ "darwin" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.57.1.tgz", - "integrity": "sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==", + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.3.tgz", + "integrity": "sha512-YtiBp4disu6V560loT6PjMdiRaWmVvDNrUunAalbiFx2ggeJwxdAsgZMcoGP17uyAsTwAj5V1niksxlHnVQ1Sw==", "cpu": [ "x64" ], @@ -873,26 +1082,15 @@ "optional": true, "os": [ "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.57.1.tgz", - "integrity": "sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.57.1.tgz", - "integrity": "sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==", + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.3.tgz", + "integrity": "sha512-yD3EkEdXk2LypPxnf/kSZHirarsI8gcPzc62SukhR9VJTyvV+F9Q/GxWNuCojc7sXyuVC4DxRGhdDK4X8VSsbw==", "cpu": [ "x64" ], @@ -901,26 +1099,15 @@ "optional": true, "os": [ "freebsd" - ] - }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.57.1.tgz", - "integrity": "sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==", - "cpu": [ - "arm" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.57.1.tgz", - "integrity": "sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==", + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.3.tgz", + "integrity": "sha512-c+8vieQbsD7HNAHKIA34w0GJ9FedFFuJGD+7E6vz7Q3uqAIugL5p45fhlsj4UaAsHpcmlqugBWMhA0/j7o0sIg==", "cpu": [ "arm" ], @@ -929,180 +1116,135 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.57.1.tgz", - "integrity": "sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.57.1.tgz", - "integrity": "sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==", + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.3.tgz", + "integrity": "sha512-50jD0uUwLvur7Zz9LHz17kaAdTPjn5wN93hEgjvmYFRZwiR7ZJYovTd5ipyWJDAnXKvZ+wgc+/Ika6dwSF5OcA==", "cpu": [ "arm64" ], "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.57.1.tgz", - "integrity": "sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==", - "cpu": [ - "loong64" + "libc": [ + "glibc" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.57.1.tgz", - "integrity": "sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==", - "cpu": [ - "loong64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.57.1.tgz", - "integrity": "sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==", + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.3.tgz", + "integrity": "sha512-BO9+oPL8K9poZJBfYPsXNtYjPE5uM3qeehT3aFcW4LITOl+iSqhp0abzjR2nWBUNjIZeKXjAEWBZ64WjNoHd6w==", "cpu": [ - "ppc64" + "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.57.1.tgz", - "integrity": "sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==", + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.3.tgz", + "integrity": "sha512-f3VpLB1vQ0Eo6ecr/6cekLnvYMFF4YBFoVGkfkvPLq1bAkbAwHYQPZKoAmG6OJyTcxxoC+AvezGx/S1obNC0Mw==", "cpu": [ "ppc64" ], "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.57.1.tgz", - "integrity": "sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==", - "cpu": [ - "riscv64" + "libc": [ + "glibc" ], - "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.57.1.tgz", - "integrity": "sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==", - "cpu": [ - "riscv64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.57.1.tgz", - "integrity": "sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==", + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.3.tgz", + "integrity": "sha512-AmurZ26Pqx/RI9N1gzEOCklkKXl927yjfXWUUS0O7Puh8ARM/Ob8qfrD3qnWksScdw6cSrW5PSHE9DyLu7+PtA==", "cpu": [ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.57.1.tgz", - "integrity": "sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==", + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.3.tgz", + "integrity": "sha512-JJpqs8bRGITDOdbkNKnlojzBabbOHrqjSvDr0IVsZObE1lBcPjxItUEY9eWIDbxaJ3cGrXPWGfGkIxFijg/URg==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.57.1.tgz", - "integrity": "sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==", + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.3.tgz", + "integrity": "sha512-rSJcdjPxzA/by/6/rYs+v+bXU7UjvnbUWz8MJb6kh6+knqB1dCrtHg0uu7C/4haqJvqdkYHQ5IGn+tCH9GLW/g==", "cpu": [ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.57.1.tgz", - "integrity": "sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==", - "cpu": [ - "x64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.57.1.tgz", - "integrity": "sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==", + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.3.tgz", + "integrity": "sha512-hQ3/PYkDJICgevvyNcVrihVeqq7k1Pp3VZ9lY+dauAYUJKO+auqApvANhvR1An9BhmqYKvW2Mu1F9u4DXSMLxQ==", "cpu": [ "arm64" ], @@ -1111,54 +1253,51 @@ "optional": true, "os": [ "openharmony" - ] - }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.57.1.tgz", - "integrity": "sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.57.1.tgz", - "integrity": "sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==", + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.3.tgz", + "integrity": "sha512-Elcv/BtML9lXrV6JuKITc/grN2kYV9gjsQpW8Jfw4ioK0TOkjBjye0nnyqQNy9STNaI20lXNaQBRrD5gSgR0Yg==", "cpu": [ - "ia32" + "wasm32" ], "dev": true, "license": "MIT", "optional": true, - "os": [ - "win32" - ] + "dependencies": { + "@emnapi/core": "1.11.1", + "@emnapi/runtime": "1.11.1", + "@napi-rs/wasm-runtime": "^1.1.6" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.57.1.tgz", - "integrity": "sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==", + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.3.tgz", + "integrity": "sha512-2DrEfhluH9yhiaFApmsjsjwrSYbNcY1oFTzYSP1a535jDbV98zCFanA/96TBUd0iDFcxGmw9QRExwGCXz3U+/g==", "cpu": [ - "x64" + "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.57.1.tgz", - "integrity": "sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==", + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.3.tgz", + "integrity": "sha512-OL4OMk7UPXOeVGGd3qo5zJyPIljf4AFgk5QAkPPS+OoLuOOozhuaQGC18MxVTnw/06q93gShAJzlwnSCY9YtqA==", "cpu": [ "x64" ], @@ -1167,29 +1306,68 @@ "optional": true, "os": [ "win32" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@sinclair/typebox": { - "version": "0.27.10", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz", - "integrity": "sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==", + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", "dev": true, "license": "MIT" }, - "node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", "dev": true, "license": "MIT" }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "node_modules/@tybys/wasm-util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", + "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", "dev": true, - "license": "MIT" - }, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/node": { "version": "20.19.33", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.33.tgz", @@ -1200,104 +1378,145 @@ "undici-types": "~6.21.0" } }, - "node_modules/@vitest/expect": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-1.6.1.tgz", - "integrity": "sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog==", + "node_modules/@vitest/coverage-v8": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.9.tgz", + "integrity": "sha512-G9/lgqibheLVBDRuya45EbsEXTYcWoSG+TLg7i2axuzx0Eq62eXn+aWXyaVdV5vKvFSWd6ywcX8hA7la9Pvu8g==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "1.6.1", - "@vitest/utils": "1.6.1", - "chai": "^4.3.10" + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.1.9", + "ast-v8-to-istanbul": "^1.0.0", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.2", + "obug": "^2.1.1", + "std-env": "^4.0.0-rc.1", + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.1.9", + "vitest": "4.1.9" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } } }, - "node_modules/@vitest/runner": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-1.6.1.tgz", - "integrity": "sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA==", + "node_modules/@vitest/expect": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.9.tgz", + "integrity": "sha512-vl/rYsUKcBr3SnQn166+XR5ZQcgMx3DQhFWdfli/cWpLnLUmbxZvyrJZotLFUryib+LtArYMSTJ5RbQ57ZqrlA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "1.6.1", - "p-limit": "^5.0.0", - "pathe": "^1.1.1" + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/runner/node_modules/p-limit": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", - "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", + "node_modules/@vitest/mocker": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.9.tgz", + "integrity": "sha512-EVkXzBjrPGM+cK8/ANWgBrkUCfJfb38/EfTSO8h7pWvKkyPkpWxvR7BkD2MyItMF62C97zAEoqdpUixwR/e+Rw==", "dev": true, "license": "MIT", "dependencies": { - "yocto-queue": "^1.0.0" + "@vitest/spy": "4.1.9", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" }, - "engines": { - "node": ">=18" + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.9.tgz", + "integrity": "sha512-s0iufns3iIFitdgm+YR7g1whCAaGtXz459VS9/PqyKDEEFgYIhsHOQmXgIgDuYCt7DeQmiZT0Qe2OA2p4ZPu5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "url": "https://opencollective.com/vitest" } }, - "node_modules/@vitest/runner/node_modules/yocto-queue": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", - "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "node_modules/@vitest/runner": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.9.tgz", + "integrity": "sha512-KXLMDtc7oe70+3mJfGrPUWPesswH+3sTxAMAMl8DG7I8IUQT4XW718dY5ID3vPUcmlu27CcKfY4P3h3I29SLJg==", "dev": true, "license": "MIT", - "engines": { - "node": ">=12.20" + "dependencies": { + "@vitest/utils": "4.1.9", + "pathe": "^2.0.3" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/snapshot": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.6.1.tgz", - "integrity": "sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ==", + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.9.tgz", + "integrity": "sha512-Jc7RKGNBo8Z28WYIm0Niej4xdSPByRf6mU58VpHQkd6Zh05rlnA+twjbK5HyeIGHxrzsc3mJgS43uM0CZKzaIA==", "dev": true, "license": "MIT", "dependencies": { - "magic-string": "^0.30.5", - "pathe": "^1.1.1", - "pretty-format": "^29.7.0" + "@vitest/pretty-format": "4.1.9", + "@vitest/utils": "4.1.9", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/spy": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-1.6.1.tgz", - "integrity": "sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw==", + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.9.tgz", + "integrity": "sha512-fHpsS6mIi+PiEW+vcRVOMkX1oSaPKne3VOclSFICPcGOmfKgXPU5iAah+wcNcj2xPrCCmfq99IDGf+EojhhvhA==", "dev": true, "license": "MIT", - "dependencies": { - "tinyspy": "^2.2.0" - }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/utils": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-1.6.1.tgz", - "integrity": "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g==", + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.9.tgz", + "integrity": "sha512-A51o8ymO5PpqlWNnBP9ZHPXDIpuMtTLlGSjN7la4US+LJzoUMyhwjA5QXlm39JexgwHKW4Xjs8Z2d3dLCXOeuA==", "dev": true, "license": "MIT", "dependencies": { - "diff-sequences": "^29.6.3", - "estree-walker": "^3.0.3", - "loupe": "^2.3.7", - "pretty-format": "^29.7.0" + "@vitest/pretty-format": "4.1.9", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" @@ -1326,19 +1545,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/acorn-walk": { - "version": "8.3.5", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", - "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.11.0" - }, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -1356,19 +1562,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -1377,13 +1570,25 @@ "license": "Python-2.0" }, "node_modules/assertion-error": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", - "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", "dev": true, "license": "MIT", "engines": { - "node": "*" + "node": ">=12" + } + }, + "node_modules/ast-v8-to-istanbul": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.4.tgz", + "integrity": "sha512-0bC0/4bTSrnwdhU3IsZDwEdojvuPrSg59OYZfKsLRtJZ0u8VBx9DebfqqG8bRdCC0I7vjgxmPi41P0lpkhJHtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" } }, "node_modules/balanced-match": { @@ -1393,6 +1598,27 @@ "dev": true, "license": "MIT" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -1404,16 +1630,6 @@ "concat-map": "0.0.1" } }, - "node_modules/cac": { - "version": "6.7.14", - "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", - "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1425,22 +1641,13 @@ } }, "node_modules/chai": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz", - "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", - "dependencies": { - "assertion-error": "^1.1.0", - "check-error": "^1.0.3", - "deep-eql": "^4.1.3", - "get-func-name": "^2.0.2", - "loupe": "^2.3.6", - "pathval": "^1.1.1", - "type-detect": "^4.1.0" - }, "engines": { - "node": ">=4" + "node": ">=18" } }, "node_modules/chalk": { @@ -1455,19 +1662,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/check-error": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz", - "integrity": "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==", - "dev": true, - "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.2" - }, - "engines": { - "node": "*" - } - }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -1504,10 +1698,10 @@ "dev": true, "license": "MIT" }, - "node_modules/confbox": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/confbox/-/confbox-0.1.8.tgz", - "integrity": "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==", + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true, "license": "MIT" }, @@ -1544,19 +1738,6 @@ } } }, - "node_modules/deep-eql": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz", - "integrity": "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-detect": "^4.0.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1564,16 +1745,22 @@ "dev": true, "license": "MIT" }, - "node_modules/diff-sequences": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", - "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", - "dev": true, - "license": "MIT", + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + "node": ">=8" } }, + "node_modules/es-module-lexer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.2.0.tgz", + "integrity": "sha512-3lGxdTXCLfe1MYfTz1y2ksAAUM4NAOP6rPEjxGJVKO7TZ5+tvHCaQWGpC4Y3IXvW3ece0Cz1cIP4FWBxOnGCTQ==", + "dev": true, + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", @@ -1826,28 +2013,14 @@ "node": ">=0.10.0" } }, - "node_modules/execa": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-8.0.1.tgz", - "integrity": "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg==", + "node_modules/expect-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", + "integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==", "dev": true, - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^8.0.1", - "human-signals": "^5.0.0", - "is-stream": "^3.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^5.1.0", - "onetime": "^6.0.0", - "signal-exit": "^4.1.0", - "strip-final-newline": "^3.0.0" - }, + "license": "Apache-2.0", "engines": { - "node": ">=16.17" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" + "node": ">=12.0.0" } }, "node_modules/fast-deep-equal": { @@ -1871,6 +2044,24 @@ "dev": true, "license": "MIT" }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -1937,29 +2128,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/get-func-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", - "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/get-stream": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-8.0.1.tgz", - "integrity": "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/get-tsconfig": { "version": "4.13.6", "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.6.tgz", @@ -2009,15 +2177,12 @@ "node": ">=8" } }, - "node_modules/human-signals": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-5.0.0.tgz", - "integrity": "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==", + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=16.17.0" - } + "license": "MIT" }, "node_modules/ignore": { "version": "5.3.2", @@ -2079,19 +2244,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", - "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -2099,10 +2251,59 @@ "dev": true, "license": "ISC" }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", - "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", "dev": true, "license": "MIT" }, @@ -2164,56 +2365,302 @@ "node": ">= 0.8.0" } }, - "node_modules/local-pkg": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/local-pkg/-/local-pkg-0.5.1.tgz", - "integrity": "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ==", + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", "dev": true, - "license": "MIT", + "license": "MPL-2.0", "dependencies": { - "mlly": "^1.7.3", - "pkg-types": "^1.2.1" + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], "engines": { - "node": ">=14" + "node": ">= 12.0.0" }, "funding": { - "url": "https://github.com/sponsors/antfu" + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^5.0.0" - }, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], "engines": { - "node": ">=10" + "node": ">= 12.0.0" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], "dev": true, - "license": "MIT" + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } }, - "node_modules/loupe": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.7.tgz", - "integrity": "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==", + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", "dev": true, "license": "MIT", "dependencies": { - "get-func-name": "^2.0.1" + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -2224,21 +2671,29 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "node_modules/magicast": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.3.tgz", + "integrity": "sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==", "dev": true, - "license": "MIT" + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.3", + "@babel/types": "^7.29.0", + "source-map-js": "^1.2.1" + } }, - "node_modules/mimic-fn": { + "node_modules/make-dir": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", - "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", "dev": true, "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, "engines": { - "node": ">=12" + "node": ">=10" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -2257,26 +2712,6 @@ "node": "*" } }, - "node_modules/mlly": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.0.tgz", - "integrity": "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.15.0", - "pathe": "^2.0.3", - "pkg-types": "^1.3.1", - "ufo": "^1.6.1" - } - }, - "node_modules/mlly/node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2285,9 +2720,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "version": "3.3.15", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", + "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", "dev": true, "funding": [ { @@ -2310,49 +2745,18 @@ "dev": true, "license": "MIT" }, - "node_modules/npm-run-path": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", - "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^4.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/npm-run-path/node_modules/path-key": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", - "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/onetime": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", - "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "node_modules/obug": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.3.tgz", + "integrity": "sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==", "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], "license": "MIT", - "dependencies": { - "mimic-fn": "^4.0.0" - }, "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": ">=12.20.0" } }, "node_modules/optionator": { @@ -2439,22 +2843,12 @@ } }, "node_modules/pathe": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", - "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", "dev": true, "license": "MIT" }, - "node_modules/pathval": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", - "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -2462,29 +2856,23 @@ "dev": true, "license": "ISC" }, - "node_modules/pkg-types": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/pkg-types/-/pkg-types-1.3.1.tgz", - "integrity": "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==", + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", - "dependencies": { - "confbox": "^0.1.8", - "mlly": "^1.7.4", - "pathe": "^2.0.1" + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/pkg-types/node_modules/pathe": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", - "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", - "dev": true, - "license": "MIT" - }, "node_modules/postcss": { - "version": "8.5.6", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", - "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "version": "8.5.16", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", + "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", "dev": true, "funding": [ { @@ -2502,7 +2890,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.11", + "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -2520,21 +2908,6 @@ "node": ">= 0.8.0" } }, - "node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -2545,13 +2918,6 @@ "node": ">=6" } }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -2572,49 +2938,51 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, - "node_modules/rollup": { - "version": "4.57.1", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.57.1.tgz", - "integrity": "sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==", + "node_modules/rolldown": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.3.tgz", + "integrity": "sha512-1F1eEtUBtFvcGm1HQ9TiUIUHPQG7mSAODrhIzjxoUEFuo8OcbrGLiVLkevNgj84TE4lnHvnumwFjhJO5Eu135g==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.8" + "@oxc-project/types": "=0.137.0", + "@rolldown/pluginutils": "^1.0.0" }, "bin": { - "rollup": "dist/bin/rollup" + "rolldown": "bin/cli.mjs" }, "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" + "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.57.1", - "@rollup/rollup-android-arm64": "4.57.1", - "@rollup/rollup-darwin-arm64": "4.57.1", - "@rollup/rollup-darwin-x64": "4.57.1", - "@rollup/rollup-freebsd-arm64": "4.57.1", - "@rollup/rollup-freebsd-x64": "4.57.1", - "@rollup/rollup-linux-arm-gnueabihf": "4.57.1", - "@rollup/rollup-linux-arm-musleabihf": "4.57.1", - "@rollup/rollup-linux-arm64-gnu": "4.57.1", - "@rollup/rollup-linux-arm64-musl": "4.57.1", - "@rollup/rollup-linux-loong64-gnu": "4.57.1", - "@rollup/rollup-linux-loong64-musl": "4.57.1", - "@rollup/rollup-linux-ppc64-gnu": "4.57.1", - "@rollup/rollup-linux-ppc64-musl": "4.57.1", - "@rollup/rollup-linux-riscv64-gnu": "4.57.1", - "@rollup/rollup-linux-riscv64-musl": "4.57.1", - "@rollup/rollup-linux-s390x-gnu": "4.57.1", - "@rollup/rollup-linux-x64-gnu": "4.57.1", - "@rollup/rollup-linux-x64-musl": "4.57.1", - "@rollup/rollup-openbsd-x64": "4.57.1", - "@rollup/rollup-openharmony-arm64": "4.57.1", - "@rollup/rollup-win32-arm64-msvc": "4.57.1", - "@rollup/rollup-win32-ia32-msvc": "4.57.1", - "@rollup/rollup-win32-x64-gnu": "4.57.1", - "@rollup/rollup-win32-x64-msvc": "4.57.1", - "fsevents": "~2.3.2" + "@rolldown/binding-android-arm64": "1.1.3", + "@rolldown/binding-darwin-arm64": "1.1.3", + "@rolldown/binding-darwin-x64": "1.1.3", + "@rolldown/binding-freebsd-x64": "1.1.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.1.3", + "@rolldown/binding-linux-arm64-gnu": "1.1.3", + "@rolldown/binding-linux-arm64-musl": "1.1.3", + "@rolldown/binding-linux-ppc64-gnu": "1.1.3", + "@rolldown/binding-linux-s390x-gnu": "1.1.3", + "@rolldown/binding-linux-x64-gnu": "1.1.3", + "@rolldown/binding-linux-x64-musl": "1.1.3", + "@rolldown/binding-openharmony-arm64": "1.1.3", + "@rolldown/binding-wasm32-wasi": "1.1.3", + "@rolldown/binding-win32-arm64-msvc": "1.1.3", + "@rolldown/binding-win32-x64-msvc": "1.1.3" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" } }, "node_modules/shebang-command": { @@ -2647,19 +3015,6 @@ "dev": true, "license": "ISC" }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -2678,25 +3033,12 @@ "license": "MIT" }, "node_modules/std-env": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", - "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", "dev": true, "license": "MIT" }, - "node_modules/strip-final-newline": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", - "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -2710,19 +3052,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-literal": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-2.1.1.tgz", - "integrity": "sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-tokens": "^9.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -2743,26 +3072,51 @@ "dev": true, "license": "MIT" }, - "node_modules/tinypool": { - "version": "0.8.4", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-0.8.4.tgz", - "integrity": "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==", + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", "dev": true, "license": "MIT", "engines": { - "node": ">=14.0.0" + "node": ">=18" } }, - "node_modules/tinyspy": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-2.2.1.tgz", - "integrity": "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==", + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", - "engines": { + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { "node": ">=14.0.0" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, "node_modules/tsx": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", @@ -2796,16 +3150,6 @@ "node": ">= 0.8.0" } }, - "node_modules/type-detect": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz", - "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -2820,13 +3164,6 @@ "node": ">=14.17" } }, - "node_modules/ufo": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.3.tgz", - "integrity": "sha512-yDJTmhydvl5lJzBmy/hyOAA0d+aqCBuwl818haVdYCRrWV84o7YyeVm4QlVHStqNrrJSTb6jKuFAVqAFsr+K3Q==", - "dev": true, - "license": "MIT" - }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -2845,21 +3182,23 @@ } }, "node_modules/vite": { - "version": "5.4.21", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", - "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.0.tgz", + "integrity": "sha512-BuJcQK/56NQTWDGn4ABea3q4SSBdNPWwNZKTkkUpcMPnLoquSYH8llRtSUIgoL1KSCpHt5eghLShn50mH36y7Q==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.21.3", - "postcss": "^8.4.43", - "rollup": "^4.20.0" + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.15", + "rolldown": "~1.1.2", + "tinyglobby": "^0.2.17" }, "bin": { "vite": "bin/vite.js" }, "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": "^20.19.0 || >=22.12.0" }, "funding": { "url": "https://github.com/vitejs/vite?sponsor=1" @@ -2868,23 +3207,33 @@ "fsevents": "~2.3.3" }, "peerDependencies": { - "@types/node": "^18.0.0 || >=20.0.0", - "less": "*", - "lightningcss": "^1.21.0", - "sass": "*", - "sass-embedded": "*", - "stylus": "*", - "sugarss": "*", - "terser": "^5.4.0" + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.3.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" }, "peerDependenciesMeta": { "@types/node": { "optional": true }, - "less": { + "@vitejs/devtools": { "optional": true }, - "lightningcss": { + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { "optional": true }, "sass": { @@ -2901,515 +3250,89 @@ }, "terser": { "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true } } }, - "node_modules/vite-node": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-1.6.1.tgz", - "integrity": "sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA==", - "dev": true, - "license": "MIT", - "dependencies": { - "cac": "^6.7.14", - "debug": "^4.3.4", - "pathe": "^1.1.1", - "picocolors": "^1.0.0", - "vite": "^5.0.0" - }, - "bin": { - "vite-node": "vite-node.mjs" - }, - "engines": { - "node": "^18.0.0 || >=20.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/vite/node_modules/@esbuild/aix-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", - "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", - "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", - "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", - "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", - "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", - "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", - "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", - "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", - "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", - "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", - "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-loong64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", - "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-mips64el": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", - "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", - "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-riscv64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", - "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-s390x": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", - "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", - "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/netbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", - "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/openbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", - "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/sunos-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", - "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", - "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", - "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", - "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/esbuild": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", - "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=12" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.21.5", - "@esbuild/android-arm": "0.21.5", - "@esbuild/android-arm64": "0.21.5", - "@esbuild/android-x64": "0.21.5", - "@esbuild/darwin-arm64": "0.21.5", - "@esbuild/darwin-x64": "0.21.5", - "@esbuild/freebsd-arm64": "0.21.5", - "@esbuild/freebsd-x64": "0.21.5", - "@esbuild/linux-arm": "0.21.5", - "@esbuild/linux-arm64": "0.21.5", - "@esbuild/linux-ia32": "0.21.5", - "@esbuild/linux-loong64": "0.21.5", - "@esbuild/linux-mips64el": "0.21.5", - "@esbuild/linux-ppc64": "0.21.5", - "@esbuild/linux-riscv64": "0.21.5", - "@esbuild/linux-s390x": "0.21.5", - "@esbuild/linux-x64": "0.21.5", - "@esbuild/netbsd-x64": "0.21.5", - "@esbuild/openbsd-x64": "0.21.5", - "@esbuild/sunos-x64": "0.21.5", - "@esbuild/win32-arm64": "0.21.5", - "@esbuild/win32-ia32": "0.21.5", - "@esbuild/win32-x64": "0.21.5" - } - }, "node_modules/vitest": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-1.6.1.tgz", - "integrity": "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "1.6.1", - "@vitest/runner": "1.6.1", - "@vitest/snapshot": "1.6.1", - "@vitest/spy": "1.6.1", - "@vitest/utils": "1.6.1", - "acorn-walk": "^8.3.2", - "chai": "^4.3.10", - "debug": "^4.3.4", - "execa": "^8.0.1", - "local-pkg": "^0.5.0", - "magic-string": "^0.30.5", - "pathe": "^1.1.1", - "picocolors": "^1.0.0", - "std-env": "^3.5.0", - "strip-literal": "^2.0.0", - "tinybench": "^2.5.1", - "tinypool": "^0.8.3", - "vite": "^5.0.0", - "vite-node": "1.6.1", - "why-is-node-running": "^2.2.2" + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.9.tgz", + "integrity": "sha512-nE3/LEyc0z87uHYLZebqCUOaJr2hdtuPp7BQ4BosVFnfltxgAvMG08NyrSGlPpOUWvR27c5flSmYFTNr78L9GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.9", + "@vitest/mocker": "4.1.9", + "@vitest/pretty-format": "4.1.9", + "@vitest/runner": "4.1.9", + "@vitest/snapshot": "4.1.9", + "@vitest/spy": "4.1.9", + "@vitest/utils": "4.1.9", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" }, "bin": { "vitest": "vitest.mjs" }, "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "@edge-runtime/vm": "*", - "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "1.6.1", - "@vitest/ui": "1.6.1", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.9", + "@vitest/browser-preview": "4.1.9", + "@vitest/browser-webdriverio": "4.1.9", + "@vitest/coverage-istanbul": "4.1.9", + "@vitest/coverage-v8": "4.1.9", + "@vitest/ui": "4.1.9", "happy-dom": "*", - "jsdom": "*" + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "@edge-runtime/vm": { "optional": true }, + "@opentelemetry/api": { + "optional": true + }, "@types/node": { "optional": true }, - "@vitest/browser": { + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { "optional": true }, "@vitest/ui": { @@ -3420,6 +3343,9 @@ }, "jsdom": { "optional": true + }, + "vite": { + "optional": false } } }, @@ -3476,9 +3402,9 @@ } }, "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", "license": "ISC", "bin": { "yaml": "bin.mjs" diff --git a/tests/package.json b/tests/package.json index 8f9b7b74..82b5fdba 100644 --- a/tests/package.json +++ b/tests/package.json @@ -21,7 +21,7 @@ "zod": "^3.23.8" }, "devDependencies": { - "@microsoft/vally": "^0.6.0", + "@microsoft/vally": "^0.7.0", "@types/node": "^20.14.0", "@vitest/coverage-v8": "^4.1.9", "eslint": "^9.5.0", @@ -32,5 +32,6 @@ }, "engines": { "node": ">=20.0.0" - } -} \ No newline at end of file + }, + "packageManager": "pnpm@11.10.0+sha512.0b7f8b98060031904c017e3a41eb187a16d40eeb829b95c4f8cb03681761fc4ab53dd219115b9b447f4dce1a05a214764461e7d3703392a9f32f9511ce8c86c8" +} diff --git a/tests/pnpm-lock.yaml b/tests/pnpm-lock.yaml index bc23d1e8..cb2b3324 100644 --- a/tests/pnpm-lock.yaml +++ b/tests/pnpm-lock.yaml @@ -25,8 +25,8 @@ importers: version: 3.25.76 devDependencies: '@microsoft/vally': - specifier: ^0.6.0 - version: 0.6.0 + specifier: ^0.7.0 + version: 0.7.0 '@types/node': specifier: ^20.14.0 version: 20.19.30 @@ -47,7 +47,7 @@ importers: version: 8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2) vitest: specifier: ^4.1.9 - version: 4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) + version: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) packages: @@ -291,24 +291,28 @@ packages: resolution: {integrity: sha512-Krg/3ZWxXB7Dw4VOLZEZrYmqc39Yvz9M1K9SPOfjpEy2SFnF/KVLaFt/6E1uYdjgvJ7BmocfVcFYp0hUmy5Axw==} cpu: [arm64] os: [linux] + libc: [glibc] hasBin: true '@github/copilot-linux-x64@1.0.64': resolution: {integrity: sha512-2k9FGppYnxHLwVH+TVCf13JfjSlvS15wsZM5xYxEFlqL/CIBvTK7yn5p7M+P1dWoTUpCmavKG3601BtHPenrRg==} cpu: [x64] os: [linux] + libc: [glibc] hasBin: true '@github/copilot-linuxmusl-arm64@1.0.64': resolution: {integrity: sha512-C+EYoMvmlUxR0YYxLkD3nwn940y0zId8z+pPl9rFO6f9heMGXYCwCZL2i2c3GW6CvGgZF6Wbzw1Kk0Gvw46F2w==} cpu: [arm64] os: [linux] + libc: [musl] hasBin: true '@github/copilot-linuxmusl-x64@1.0.64': resolution: {integrity: sha512-VSTGl7dGQDhH2ACeyYq1hYhAsMUbumpqeVe+CGo/u7fKOMrTNMTR5SB3s+zsHCskKZkJkZ2gsUWROCiDBsThEg==} cpu: [x64] os: [linux] + libc: [musl] hasBin: true '@github/copilot-sdk@1.0.3': @@ -357,8 +361,8 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} - '@microsoft/vally@0.6.0': - resolution: {integrity: sha512-b283YRDFZXUkKNKY3+1EfMBVbHrBLIs5jfUi7lIQ8N0Y10lVsNnNGkRbtTbd7tLYZajr6AhtnpIroc4RFzo1cQ==} + '@microsoft/vally@0.7.0': + resolution: {integrity: sha512-GYxSgub10SC6X1d/vVUCOorGzAMWIUN7uk2zq65pPle0CfKdWy+HiEpnJkgrXGat7kZxzx+jM9lrffW3nfKpmw==} '@napi-rs/wasm-runtime@1.1.5': resolution: {integrity: sha512-AWPoBRJ9tsnVhor4sjO7rkni+7p+2IAEFj6cx06UgP10jkQHqay/36uRV/bFkgrh18D9vb4cr8Q0Pthskgzy+Q==} @@ -366,6 +370,10 @@ packages: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + '@oxc-project/types@0.137.0': resolution: {integrity: sha512-WT+Gb24i8hmvo85AIv2oEYouEXkRlKAlT9WaCa3TfLgNCN+GhrJOGZuIlMouAh38Qe4QOx26eUOVsq70qXrywA==} @@ -404,36 +412,42 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-arm64-musl@1.1.2': resolution: {integrity: sha512-LjQP/iZLBu8o8PjIfk4x3At0/mT6h282pvz8Z5LAyhGbu/kDezyO7ea62rF5uoqmgnIYqbN/MqJ3Si3Aymi7xQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [linux] + libc: [musl] '@rolldown/binding-linux-ppc64-gnu@1.1.2': resolution: {integrity: sha512-X/7bVLWelEsbyWDUSXt7zVsTniLLPIY2n1rH58qr78l9i7MNbbxBWD8gI2vRfBWf4NUXJCUuQnfZDsp32LqsfQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [ppc64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-s390x-gnu@1.1.2': resolution: {integrity: sha512-gb6dYKW/1KDorGXyy48glEBJs/sxVSC5pcVrox/pFGV4mvwSFeg2sK5L2tRkVsVlh7kueqOgg4GEcuipJcGuKg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [s390x] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-gnu@1.1.2': resolution: {integrity: sha512-JY4w85pU3iAiJVMh5nuk4/Mh9GjMsupe8MrIN53rwxAZW64GKrWeJBuN6SxQg9QTU5uB1cxyhDzW8jqRn1EABw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [glibc] '@rolldown/binding-linux-x64-musl@1.1.2': resolution: {integrity: sha512-xvpA7o5KCYLB0Rwscmuylb1/zHHSUx4g4xilm4prC5jP76pEUlzBmMbgpbh7bVDbId4NcfT96gN5i6mE6UDaiw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [linux] + libc: [musl] '@rolldown/binding-openharmony-arm64@1.1.2': resolution: {integrity: sha512-p/ts6KBLjuk49Bp21XH77poQGt02iNz7ChgHep7tudPOaLinR/De/RHdxF8w8Yj4r/bF/bqXwH6PZrB2sA+Nvw==} @@ -822,24 +836,28 @@ packages: engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [glibc] lightningcss-linux-arm64-musl@1.32.0: resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} engines: {node: '>= 12.0.0'} cpu: [arm64] os: [linux] + libc: [musl] lightningcss-linux-x64-gnu@1.32.0: resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [glibc] lightningcss-linux-x64-musl@1.32.0: resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} engines: {node: '>= 12.0.0'} cpu: [x64] os: [linux] + libc: [musl] lightningcss-win32-arm64-msvc@1.32.0: resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} @@ -1364,9 +1382,10 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@microsoft/vally@0.6.0': + '@microsoft/vally@0.7.0': dependencies: '@github/copilot-sdk': 1.0.3 + '@opentelemetry/api': 1.9.1 js-tiktoken: 1.0.21 picomatch: 4.0.4 yaml: 2.9.0 @@ -1379,6 +1398,8 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true + '@opentelemetry/api@1.9.1': {} + '@oxc-project/types@0.137.0': {} '@rolldown/binding-android-arm64@1.1.2': @@ -1466,7 +1487,7 @@ snapshots: obug: 2.1.3 std-env: 4.1.0 tinyrainbow: 3.1.0 - vitest: 4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) + vitest: 4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/expect@4.1.9': dependencies: @@ -2001,7 +2022,7 @@ snapshots: tsx: 4.21.0 yaml: 2.8.2 - vitest@4.1.9(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)): + vitest@4.1.9(@opentelemetry/api@1.9.1)(@types/node@20.19.30)(@vitest/coverage-v8@4.1.9)(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)): dependencies: '@vitest/expect': 4.1.9 '@vitest/mocker': 4.1.9(vite@8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2)) @@ -2024,6 +2045,7 @@ snapshots: vite: 8.1.0(@types/node@20.19.30)(esbuild@0.27.2)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: + '@opentelemetry/api': 1.9.1 '@types/node': 20.19.30 '@vitest/coverage-v8': 4.1.9(vitest@4.1.9) transitivePeerDependencies: diff --git a/tests/run-all-evals.ps1 b/tests/run-all-evals.ps1 new file mode 100644 index 00000000..a76ac734 --- /dev/null +++ b/tests/run-all-evals.ps1 @@ -0,0 +1,308 @@ +<# +.SYNOPSIS + Runs all Vally evaluations across skill scenarios. + +.DESCRIPTION + Discovers eval.yaml files under the scenarios directory and runs them in parallel + using the Vally evaluation harness. Supports filtering by language and Azure service, + and optional JUnit XML output for CI integration. + +.PARAMETER ScenariosRoot + Path to the directory containing skill scenario folders. + Defaults to: /scenarios + +.PARAMETER ResultsRoot + Path to the directory where evaluation results are written. + Defaults to: /scenario-results + +.PARAMETER Workers + Number of parallel evaluation workers. Default: 5 + +.PARAMETER JUnit + If specified, emit JUnit-compatible XML output alongside the standard results. + +.PARAMETER Language + Filter evaluations to only skills matching the given language suffix(es). + Examples: -Language py, -Language py,ts, -Language rust + +.PARAMETER AzureService + Filter evaluations to only skills matching the given Azure service name(s). + Examples: -AzureService cosmos, -AzureService storage,servicebus + +.EXAMPLE + # Run all evaluations + .\run-all-evals.ps1 + +.EXAMPLE + # Run only Python evaluations + .\run-all-evals.ps1 -Language py + +.EXAMPLE + # Run only Cosmos DB evaluations with JUnit output + .\run-all-evals.ps1 -AzureService cosmos -JUnit + +.EXAMPLE + # Run storage evaluations with 10 parallel workers + .\run-all-evals.ps1 -AzureService storage -Workers 10 + +.EXAMPLE + # Run against a custom scenarios directory + .\run-all-evals.ps1 -ScenariosRoot C:\my-scenarios -ResultsRoot C:\my-results + +.EXAMPLE + # Run with verbose output + .\run-all-evals.ps1 -Verbose + +.EXAMPLE + # Run with debug information + .\run-all-evals.ps1 -Debug + +.NOTES + Supports standard PowerShell switches: -Verbose, -Debug, -InformationAction, -WarningAction, -ErrorAction +#> +[CmdletBinding()] +param( + [Parameter(HelpMessage = "Path to the directory containing skill scenario folders")] + [string]$ScenariosRoot, + + [Parameter(HelpMessage = "Path to the directory where evaluation results are written")] + [string]$ResultsRoot, + + [Parameter(HelpMessage = "Number of parallel evaluation workers")] + [int]$Workers = 5, + + [Parameter(HelpMessage = "Emit JUnit-compatible XML output")] + [switch]$JUnit, + + [Parameter(HelpMessage = "Filter by language suffix(es): py, ts, dotnet, java, rust")] + [string[]]$Language, + + [Parameter(HelpMessage = "Filter by Azure service name(s)")] + [string[]]$AzureService +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +if (-not $PSBoundParameters.ContainsKey("ScenariosRoot")) { + $ScenariosRoot = Join-Path $PSScriptRoot "scenarios" +} + +if (-not $PSBoundParameters.ContainsKey("ResultsRoot")) { + $ResultsRoot = Join-Path $PSScriptRoot "scenario-results" +} + +if (-not (Test-Path -LiteralPath $ScenariosRoot)) { + Write-Error "Scenarios root not found: $ScenariosRoot" + exit 1 +} + +$resolvedScenariosRoot = (Resolve-Path -LiteralPath $ScenariosRoot).ProviderPath +$resolvedResultsRoot = [System.IO.Path]::GetFullPath($ResultsRoot) + +$languageFilter = @($Language | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { $_.Trim().ToLowerInvariant() }) +$serviceFilter = @($AzureService | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { $_.Trim().ToLowerInvariant() }) + +$evalFiles = Get-ChildItem -Path $resolvedScenariosRoot -Filter "eval.yaml" -File -Recurse | +Where-Object { + $relativeDir = [System.IO.Path]::GetRelativePath($resolvedScenariosRoot, $_.DirectoryName).TrimStart('\', '/') + if ([string]::IsNullOrWhiteSpace($relativeDir)) { return $false } + + $skillName = ($relativeDir -split '[\\/]', 2)[0] + if ([string]::IsNullOrWhiteSpace($skillName)) { return $false } + + $languageFromSkill = "" + if ($skillName -match '-([^-]+)$') { + $languageFromSkill = $Matches[1].ToLowerInvariant() + } + + $serviceFromSkill = $skillName.ToLowerInvariant() + if ($languageFromSkill) { + $serviceFromSkill = $serviceFromSkill -replace ("-{0}$" -f [regex]::Escape($languageFromSkill)), "" + } + + $includeLanguage = $true + if ($languageFilter.Count -gt 0) { + $includeLanguage = $languageFromSkill -and ($languageFilter -contains $languageFromSkill) + } + + $includeService = $true + if ($serviceFilter.Count -gt 0) { + $includeService = $false + foreach ($svc in $serviceFilter) { + if ($serviceFromSkill -eq $svc -or $serviceFromSkill -like "*$svc*") { + $includeService = $true + break + } + } + } + + return $includeLanguage -and $includeService +} | +Sort-Object FullName + +if (-not $evalFiles) { + $filterSummary = @() + if ($languageFilter.Count -gt 0) { $filterSummary += "language=$($languageFilter -join ',')" } + if ($serviceFilter.Count -gt 0) { $filterSummary += "service=$($serviceFilter -join ',')" } + if ($filterSummary.Count -eq 0) { $filterSummary += "no filters" } + Write-Error "No eval.yaml files found under $resolvedScenariosRoot with $($filterSummary -join '; ')" + exit 1 +} + +New-Item -ItemType Directory -Path $resolvedResultsRoot -Force | Out-Null + +$customGraderPluginDir = Join-Path $PSScriptRoot "scenarios\_shared\vally\grader-plugins\rust-cargo-build-failure" +$customGraderSource = Join-Path $customGraderPluginDir "index.ts" +$customGraderPackage = Join-Path $customGraderPluginDir "package.json" +$customGraderTsConfig = Join-Path $customGraderPluginDir "tsconfig.json" +$customGraderDistEntry = Join-Path $customGraderPluginDir "dist\index.js" + +$requiresRustCustomGrader = $false +foreach ($eval in $evalFiles) { + $match = Select-String -Path $eval.FullName -Pattern 'type:\s*rust-cargo-build-failure-check' -CaseSensitive:$false -ErrorAction SilentlyContinue + if ($match) { + $requiresRustCustomGrader = $true + break + } +} + +if ($requiresRustCustomGrader) { + if (-not (Test-Path $customGraderPluginDir)) { + Write-Error "Custom grader plugin directory not found: $customGraderPluginDir" + exit 1 + } + + $customGraderRebuildReasons = @() + if (-not (Test-Path $customGraderDistEntry)) { + $customGraderRebuildReasons += "dist/index.js is missing" + } + else { + $distTime = (Get-Item $customGraderDistEntry).LastWriteTimeUtc + foreach ($sourcePath in @($customGraderSource, $customGraderTsConfig, $customGraderPackage)) { + if (Test-Path $sourcePath) { + $sourceTime = (Get-Item $sourcePath).LastWriteTimeUtc + if ($sourceTime -gt $distTime) { + $customGraderRebuildReasons += "$(Split-Path -Leaf $sourcePath) is newer than dist/index.js" + } + } + } + } + + if ($customGraderRebuildReasons.Count -gt 0) { + Write-Host "Custom grader plugin rebuild required: $($customGraderRebuildReasons -join '; ')" + + $buildOutput = $null + $buildExitCode = 1 + + $pnpmCmd = Get-Command pnpm -ErrorAction SilentlyContinue + if ($pnpmCmd) { + Write-Host "Rebuilding custom grader plugin via pnpm exec tsc..." + $buildOutput = & pnpm --dir $PSScriptRoot exec tsc --project $customGraderTsConfig 2>&1 + $buildExitCode = $LASTEXITCODE + } + else { + Write-Host "pnpm not found; rebuilding custom grader plugin via npx tsc..." + $buildOutput = & npx tsc --project $customGraderTsConfig 2>&1 + $buildExitCode = $LASTEXITCODE + } + + if ($buildExitCode -ne 0 -or -not (Test-Path $customGraderDistEntry)) { + if ($buildOutput) { + $buildOutput | ForEach-Object { Write-Error $_ } + } + Write-Error "Failed to build custom grader plugin at $customGraderPluginDir" + exit 1 + } + + Write-Host "Custom grader plugin rebuilt successfully." + } + else { + Write-Host "Custom grader plugin is up to date." + } +} + +$results = @() + +foreach ($evalFile in $evalFiles) { + $relativeDir = [System.IO.Path]::GetRelativePath($resolvedScenariosRoot, $evalFile.DirectoryName).TrimStart('\', '/') + if ([string]::IsNullOrWhiteSpace($relativeDir)) { $relativeDir = "root" } + + $safeOutDirName = $relativeDir -replace '[\\/:*?"<>|]', "_" + $scenarioOutDir = Join-Path $resolvedResultsRoot $safeOutDirName + New-Item -ItemType Directory -Path $scenarioOutDir -Force | Out-Null + + Write-Host "`n=== Running eval: $($evalFile.FullName) ===" + + $start = Get-Date + $npxArgs = @("vally", "eval", "--eval-spec", $evalFile.FullName, "--output-dir", $scenarioOutDir, "--workers", "$Workers") + if ($requiresRustCustomGrader) { $npxArgs += @("--grader-plugin", $customGraderPluginDir) } + if ($JUnit) { $npxArgs += "--junit" } + + & npx @npxArgs + $exitCode = $LASTEXITCODE + $durationSec = [math]::Round(((Get-Date) - $start).TotalSeconds, 1) + + $runDir = Get-ChildItem -Path $scenarioOutDir -Directory -ErrorAction SilentlyContinue | + Sort-Object LastWriteTimeUtc -Descending | + Select-Object -First 1 + + $summary = $null + if ($runDir) { + $jsonlPath = Join-Path $runDir.FullName "results.jsonl" + if (Test-Path -LiteralPath $jsonlPath) { + $line = Get-Content -Path $jsonlPath | + Where-Object { $_ -match '"type"\s*:\s*"run-summary"' } | + Select-Object -Last 1 + if ($line) { + try { $summary = $line | ConvertFrom-Json -Depth 20 } catch {} + } + } + } + + $passed = if ($summary) { [bool]$summary.passed } else { $exitCode -eq 0 } + + $details = "" + if ($summary -and $summary.evals) { + $detailItems = foreach ($e in $summary.evals) { + $status = if ($e.passed) { "PASS" } else { "FAIL" } + if ($e.scoringApplied -and $null -ne $e.overallScore -and $null -ne $e.threshold) { + "{0}:{1} ({2}%/{3}%)" -f $e.name, $status, ([math]::Round([double]$e.overallScore * 100, 1)), ([math]::Round([double]$e.threshold * 100, 1)) + } + elseif ($e.error) { + "{0}:{1} ({2})" -f $e.name, $status, $e.error + } + else { + "{0}:{1}" -f $e.name, $status + } + } + $details = ($detailItems -join "; ") + } + + $results += [pscustomobject]@{ + Scenario = $relativeDir + EvalSpec = $evalFile.FullName + Status = if ($passed) { "PASS" } else { "FAIL" } + ExitCode = $exitCode + DurationSec = $durationSec + RunDir = if ($runDir) { $runDir.FullName } else { "" } + Details = $details + } +} + +Write-Host "`n=== Vally evaluation summary ===" +$results | Sort-Object Scenario | Format-Table -AutoSize Scenario, Status, ExitCode, DurationSec, RunDir + +$reportPath = Join-Path $resolvedResultsRoot ("summary-{0}.csv" -f (Get-Date -Format "yyyyMMdd-HHmmss")) +$results | Export-Csv -NoTypeInformation -Path $reportPath +Write-Host "`nDetailed report saved to: $reportPath" + +$failed = $results | Where-Object { $_.Status -eq "FAIL" } +if ($failed) { + Write-Host "`n=== Failed eval details ===" + $failed | Format-Table -AutoSize Scenario, Details + exit 1 +} + +Write-Host "`nAll evaluations passed." +exit 0 diff --git a/tests/run-harness-by-language.ps1 b/tests/run-harness-by-language.ps1 index 07b33447..d26a6538 100644 --- a/tests/run-harness-by-language.ps1 +++ b/tests/run-harness-by-language.ps1 @@ -18,7 +18,7 @@ If specified, only run mock harness evaluations (skips real Copilot). If specified, only run real Copilot evaluations (skips mock). .PARAMETER ShowDetails -Show detailed output for each scenario. +Show per-scenario details, including generated code and findings. .PARAMETER OutputFile Optional path to save results as JSON. Defaults to stdout only. @@ -113,9 +113,185 @@ $results = @{ } } +function Get-DiagnosticErrorKind { + param( + [string[]]$Diagnostics + ) + + $joined = ($Diagnostics -join "`n").ToLowerInvariant() + + if ($joined -match "authentication failed|bad credentials|github cli user login|401") { + return "auth" + } + if ($joined -match "timeout|session\.idle|timed out") { + return "timeout" + } + if ($joined -match "rate limit|429|econnreset|etimedout|socket hang up|temporarily unavailable|service unavailable") { + return "transient" + } + if ($joined -match "empty or invalid|failed to parse|did not produce json output file") { + return "parse" + } + + return "fatal" +} + +function Test-HarnessAuthPreflight { + if ($env:GH_TOKEN -or $env:GITHUB_TOKEN) { + return $true + } + + try { + $null = & gh auth status 2>&1 + return $LASTEXITCODE -eq 0 + } + catch { + return $false + } +} + +function Get-HarnessModeResult { + param( + [Parameter(Mandatory = $true)] + [string]$Skill, + + [Parameter(Mandatory = $true)] + [bool]$UseMock, + + [switch]$IncludeDetails + ) + + $tempJson = [System.IO.Path]::GetTempFileName() + $args = @("harness", $Skill, "--output", "json", "--output-file", $tempJson) + + if ($UseMock) { + $args += "--mock" + } + + # Capture stdout/stderr as plain strings in case parsing fails and we need diagnostics. + $rawOutput = & pnpm @args 2>&1 | ForEach-Object { "$_" } + $exitCode = $LASTEXITCODE + + try { + if (-not (Test-Path $tempJson)) { + $kind = Get-DiagnosticErrorKind -Diagnostics @($rawOutput) + return @{ + status = "ERROR" + error_kind = $kind + exit_code = $exitCode + error = "Harness did not produce JSON output file." + diagnostics = @($rawOutput) + } + } + + $jsonText = Get-Content $tempJson -Raw + if ([string]::IsNullOrWhiteSpace($jsonText)) { + $kind = Get-DiagnosticErrorKind -Diagnostics @($rawOutput) + return @{ + status = "ERROR" + error_kind = $kind + exit_code = $exitCode + error = "Harness JSON output file was empty." + diagnostics = @($rawOutput) + } + } + + $parsed = $jsonText | ConvertFrom-Json -Depth 100 + + if ($null -eq $parsed) { + $kind = Get-DiagnosticErrorKind -Diagnostics @($rawOutput) + return @{ + status = "ERROR" + error_kind = $kind + exit_code = $exitCode + error = "Harness JSON output was empty or invalid." + diagnostics = @($rawOutput) + } + } + + if ($parsed.status -eq "ERROR") { + $combinedDiagnostics = @($rawOutput) + if ($parsed.diagnostics) { + $combinedDiagnostics += @($parsed.diagnostics) + } + + return @{ + status = "ERROR" + error_kind = if ($parsed.error_kind) { $parsed.error_kind } else { Get-DiagnosticErrorKind -Diagnostics $combinedDiagnostics } + exit_code = $exitCode + error = if ($parsed.error) { $parsed.error } else { "Harness reported an execution error." } + diagnostics = $combinedDiagnostics + } + } + + $result = @{ + status = if ($exitCode -eq 0) { "PASS" } else { "FAIL" } + exit_code = $exitCode + summary = @{ + total_scenarios = $parsed.total_scenarios + passed = $parsed.passed + failed = $parsed.failed + pass_rate = $parsed.pass_rate + avg_score = $parsed.avg_score + duration_ms = $parsed.duration_ms + } + failed_scenarios = @() + } + + foreach ($scenarioResult in @($parsed.results)) { + if (-not $scenarioResult.passed) { + $result.failed_scenarios += @{ + scenario = $scenarioResult.scenario + score = $scenarioResult.score + findings = @($scenarioResult.findings) + } + } + } + + if ($IncludeDetails) { + $result.scenarios = @() + foreach ($scenarioResult in @($parsed.results)) { + $result.scenarios += @{ + scenario = $scenarioResult.scenario + passed = $scenarioResult.passed + score = $scenarioResult.score + generated_code = $scenarioResult.generated_code + raw_response = $scenarioResult.raw_response + findings = @($scenarioResult.findings) + } + } + } + + return $result + } + catch { + $kind = Get-DiagnosticErrorKind -Diagnostics @($rawOutput) + return @{ + status = "ERROR" + error_kind = $kind + exit_code = $exitCode + error = "Failed to parse harness JSON output: $($_.Exception.Message)" + diagnostics = @($rawOutput) + } + } + finally { + Remove-Item -Path $tempJson -Force -ErrorAction SilentlyContinue + } +} + # Run harness for each skill Set-Location $testsDir +$canRunRealHarness = $true +if ($runReal) { + $canRunRealHarness = Test-HarnessAuthPreflight + if (-not $canRunRealHarness) { + Write-Host "Real harness auth preflight failed. Skipping all real-mode runs." -ForegroundColor Red + Write-Host "Run 'gh auth login' or set GH_TOKEN/GITHUB_TOKEN, then rerun." -ForegroundColor Yellow + Write-Host "" + } +} + foreach ($skill in $skills) { $skillResult = @{ name = $skill @@ -130,29 +306,25 @@ foreach ($skill in $skills) { if ($runMock) { Write-Host " [MOCK] Running..." -ForegroundColor Gray -NoNewline try { - $mockOutput = & pnpm harness $skill --mock 2>&1 - $mockSuccess = $LASTEXITCODE -eq 0 + $mockResult = Get-HarnessModeResult -Skill $skill -UseMock $true -IncludeDetails:$ShowDetails + $mockSuccess = $mockResult.status -eq "PASS" if ($mockSuccess) { Write-Host " ✓ PASS" -ForegroundColor Green $results.summary.mock.passed++ - $skillResult.mock = @{ - status = "PASS" - output = $mockOutput - } + $skillResult.mock = $mockResult } else { - Write-Host " ✗ FAIL" -ForegroundColor Red + $statusColor = if ($mockResult.status -eq "ERROR") { "Red" } else { "Red" } + Write-Host " ✗ $($mockResult.status)" -ForegroundColor $statusColor $results.summary.mock.failed++ $results.summary.mock.errors += $skill - $skillResult.mock = @{ - status = "FAIL" - output = $mockOutput - } + $skillResult.mock = $mockResult } - if ($ShowDetails) { - Write-Host $mockOutput -ForegroundColor DarkGray + if ($ShowDetails -and $skillResult.mock.summary) { + $mockSummary = $skillResult.mock.summary + Write-Host " scenarios: $($mockSummary.passed)/$($mockSummary.total_scenarios), avg score: $([math]::Round([double]$mockSummary.avg_score, 1))" -ForegroundColor DarkGray } } catch { @@ -168,31 +340,41 @@ foreach ($skill in $skills) { # Real harness if ($runReal) { + if (-not $canRunRealHarness) { + Write-Host " [REAL] Skipped (auth preflight failed)" -ForegroundColor Red + $results.summary.real.failed++ + $results.summary.real.errors += "$skill (AUTH_ERROR)" + $skillResult.real = @{ + status = "ERROR" + error_kind = "auth" + error = "Skipped due to failed auth preflight. Run 'gh auth login' or set GH_TOKEN/GITHUB_TOKEN." + } + Write-Host "" + $results.skills += $skillResult + continue + } + Write-Host " [REAL] Running..." -ForegroundColor Gray -NoNewline try { - $realOutput = & pnpm harness $skill 2>&1 - $realSuccess = $LASTEXITCODE -eq 0 + $realResult = Get-HarnessModeResult -Skill $skill -UseMock $false -IncludeDetails:$ShowDetails + $realSuccess = $realResult.status -eq "PASS" if ($realSuccess) { Write-Host " ✓ PASS" -ForegroundColor Green $results.summary.real.passed++ - $skillResult.real = @{ - status = "PASS" - output = $realOutput - } + $skillResult.real = $realResult } else { - Write-Host " ✗ FAIL" -ForegroundColor Red + $statusColor = if ($realResult.status -eq "ERROR") { "Red" } else { "Red" } + Write-Host " ✗ $($realResult.status)" -ForegroundColor $statusColor $results.summary.real.failed++ $results.summary.real.errors += $skill - $skillResult.real = @{ - status = "FAIL" - output = $realOutput - } + $skillResult.real = $realResult } - if ($ShowDetails) { - Write-Host $realOutput -ForegroundColor DarkGray + if ($ShowDetails -and $skillResult.real.summary) { + $realSummary = $skillResult.real.summary + Write-Host " scenarios: $($realSummary.passed)/$($realSummary.total_scenarios), avg score: $([math]::Round([double]$realSummary.avg_score, 1))" -ForegroundColor DarkGray } } catch { diff --git a/tests/run-skill-experiments.ps1 b/tests/run-skill-experiments.ps1 new file mode 100644 index 00000000..3820535a --- /dev/null +++ b/tests/run-skill-experiments.ps1 @@ -0,0 +1,1636 @@ +#Requires -Version 7.0 +<# +.SYNOPSIS +Run vally skill effectiveness experiments for specified skills. + +.DESCRIPTION +Executes skill_effectiveness_experiment.yaml files for all or specified skills. +Each experiment runs two variants (baseline without skill, skill with skill context) and +stores results organized by timestamp. + +Use -SkillPattern to filter by language (e.g., '*-py' for Python, '*-rs' for Rust) +or by service (e.g., 'azure-cosmos*' for Cosmos DB). + +Results can be compared using compare-experiment.mjs to analyze skill impact on performance. + +.PARAMETER ScenariosRoot +Root directory containing skill scenario subdirectories. +Defaults to: /scenarios + +.PARAMETER ResultsRoot +Root directory where experiment results will be stored. +Defaults to: /vally-experiment-results + +.PARAMETER SkillPattern +Filter which skills to run. Supports wildcards. +Defaults to '*-py' (all Python skills). +Examples: -SkillPattern '*-rust', -SkillPattern 'azure-ai-*' + +.PARAMETER Workers +Number of parallel experiment runs. Defaults to 1 (sequential). + +.PARAMETER DryRun +If set, shows what experiments would be run without actually running them. + +.PARAMETER Compare +If set, runs compare-experiment.mjs after completion to generate A/B reports. + +.PARAMETER AnalysisOnly +If set, skips vally execution and regenerates narratives/comparisons from existing outputs. + +.PARAMETER ExperimentOutputDir +Existing experiment output root to analyze when using -AnalysisOnly. +If omitted in analysis-only mode, the latest directory under ResultsRoot is used. + +.PARAMETER CopilotMaxAttempts +Maximum retries per copilot invocation path for narrative generation. + +.PARAMETER NoNodeLoaderFallback +Disable node-loader fallback and use only documented copilot invocation methods. + +.EXAMPLE +./run-skill-experiments.ps1 +# Run all Python skill experiments sequentially + +.EXAMPLE +./run-skill-experiments.ps1 -SkillPattern '*-rust' -Workers 4 +# Run all Rust skill experiments with 4 parallel workers + +.EXAMPLE +./run-skill-experiments.ps1 -SkillPattern 'azure-cosmos*' -Verbose +# Run only Cosmos DB skill experiments with verbose output + +.EXAMPLE +./run-skill-experiments.ps1 -Workers 5 -Compare +# Run all (Python) experiments with 5 parallel workers, then generate comparison reports + +.EXAMPLE +./run-skill-experiments.ps1 -DryRun +# Preview which experiments would be run + +.EXAMPLE +./run-skill-experiments.ps1 -AnalysisOnly -ExperimentOutputDir "q:\src\skills\tests\vally-experiment-results\2026-07-06T17-00-00-000Z" +# Regenerate narratives/comparisons from an existing experiment output without rerunning vally +#> + +[CmdletBinding()] +param( + [Parameter(Mandatory = $false)] + [string]$ScenariosRoot = (Join-Path $PSScriptRoot "scenarios"), + + [Parameter(Mandatory = $false)] + [string]$ResultsRoot = (Join-Path $PSScriptRoot "vally-experiment-results"), + + [Parameter(Mandatory = $false)] + [string]$SkillPattern = "*-py", + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 128)] + [int]$Workers = 1, + + [Parameter(Mandatory = $false)] + [switch]$DryRun, + + [Parameter(Mandatory = $false)] + [switch]$Compare + + , + [Parameter(Mandatory = $false)] + [switch]$AnalysisOnly + + , + [Parameter(Mandatory = $false)] + [string]$ExperimentOutputDir + + , + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10)] + [int]$CopilotMaxAttempts = 2, + + [Parameter(Mandatory = $false)] + [switch]$NoNodeLoaderFallback +) + +$ErrorActionPreference = "Stop" + +function Get-ExperimentOutputDirectory { + param( + [Parameter(Mandatory = $true)] + [string]$OutputText + ) + + $outputMatches = [regex]::Matches($OutputText, 'Output:\s*(.+)') + if ($outputMatches.Count -eq 0) { + return $null + } + + for ($i = $outputMatches.Count - 1; $i -ge 0; $i--) { + $candidate = $outputMatches[$i].Groups[1].Value.Trim() + if (Test-Path $candidate) { + return $candidate + } + } + + return $outputMatches[$outputMatches.Count - 1].Groups[1].Value.Trim() +} + +function Get-VariantDigest { + param( + [Parameter(Mandatory = $true)] + [string]$ResultsJsonlPath + ) + + if (-not (Test-Path $ResultsJsonlPath)) { + return $null + } + + $records = @() + foreach ($line in (Get-Content -Path $ResultsJsonlPath -ErrorAction SilentlyContinue)) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + try { + $obj = $line | ConvertFrom-Json + if ($obj.type -eq "trial-result") { + $records += $obj + } + } + catch { + continue + } + } + + if ($records.Count -eq 0) { + return $null + } + + $variantName = $records[0].variant + $evalName = $records[0].evalName + $model = $records[0].model + + $scores = @() + $tokens = @() + $turns = @() + $toolCalls = @() + $wallTimesMs = @() + $errors = @() + $skillActivations = @() + $toolBreakdownTotals = @{} + $skillBreakdownTotals = @{} + $failedGraderBreakdown = @{} + $timeoutGraderBreakdown = @{} + $totalFailedGraders = 0 + $totalTimeoutGraders = 0 + $stimuli = @() + + foreach ($record in $records) { + if ($record.gradeResult -and $null -ne $record.gradeResult.score) { + $scores += [double]$record.gradeResult.score + } + + $traj = $record.trajectory + $metrics = $traj.metrics + if ($metrics) { + if ($metrics.tokenUsage -and $null -ne $metrics.tokenUsage.totalTokens) { + $tokens += [int]$metrics.tokenUsage.totalTokens + } + if ($null -ne $metrics.turnCount) { + $turns += [int]$metrics.turnCount + } + if ($null -ne $metrics.toolCallCount) { + $toolCalls += [int]$metrics.toolCallCount + } + if ($null -ne $metrics.wallTimeMs) { + $wallTimesMs += [int]$metrics.wallTimeMs + } + if ($null -ne $metrics.errorCount) { + $errors += [int]$metrics.errorCount + } + if ($null -ne $metrics.skillActivationCount) { + $skillActivations += [int]$metrics.skillActivationCount + } + + if ($metrics.toolCallBreakdown) { + $metrics.toolCallBreakdown.psobject.Properties | ForEach-Object { + if (-not $toolBreakdownTotals.ContainsKey($_.Name)) { + $toolBreakdownTotals[$_.Name] = 0 + } + $toolBreakdownTotals[$_.Name] += [int]$_.Value + } + } + + if ($metrics.skillActivationBreakdown) { + $metrics.skillActivationBreakdown.psobject.Properties | ForEach-Object { + if (-not $skillBreakdownTotals.ContainsKey($_.Name)) { + $skillBreakdownTotals[$_.Name] = 0 + } + $skillBreakdownTotals[$_.Name] += [int]$_.Value + } + } + } + + $stimulus = $traj.stimulus + $graderOutcomes = @() + if ($record.gradeResult -and $record.gradeResult.details) { + foreach ($detail in $record.gradeResult.details) { + $graderEvidence = if ($null -ne $detail.evidence) { [string]$detail.evidence } else { "" } + $isTimeout = (-not [bool]$detail.passed) -and ($graderEvidence -match '(?i)\btime(d)?\s*out\b|\btimeout\b') + + if (-not [bool]$detail.passed) { + $totalFailedGraders++ + + if (-not $failedGraderBreakdown.ContainsKey($detail.name)) { + $failedGraderBreakdown[$detail.name] = 0 + } + $failedGraderBreakdown[$detail.name] += 1 + + if ($isTimeout) { + $totalTimeoutGraders++ + if (-not $timeoutGraderBreakdown.ContainsKey($detail.name)) { + $timeoutGraderBreakdown[$detail.name] = 0 + } + $timeoutGraderBreakdown[$detail.name] += 1 + } + } + + $graderOutcomes += [ordered]@{ + name = $detail.name + passed = [bool]$detail.passed + score = if ($null -ne $detail.score) { [double]$detail.score } else { $null } + evidence = $detail.evidence + isTimeout = $isTimeout + } + } + } + + $outputPreview = "" + if ($traj -and $traj.output) { + $outputString = [string]$traj.output + $maxLen = 800 + $outputPreview = if ($outputString.Length -gt $maxLen) { $outputString.Substring(0, $maxLen) } else { $outputString } + } + + $stimuli += [ordered]@{ + name = $record.stimulus + passed = [bool]$record.gradeResult.passed + scorePct = if ($null -ne $record.gradeResult.score) { [math]::Round(([double]$record.gradeResult.score * 100), 1) } else { $null } + durationSec = [math]::Round(([double]$record.durationMs / 1000), 1) + prompt = if ($stimulus) { $stimulus.prompt } else { $null } + rubric = if ($stimulus) { $stimulus.rubric } else { @() } + toolCallBreakdown = if ($metrics) { $metrics.toolCallBreakdown } else { $null } + graderOutcomes = $graderOutcomes + outputPreview = $outputPreview + } + } + + return [ordered]@{ + variant = $variantName + eval = $evalName + model = $model + trialCount = $records.Count + avgScorePct = if ($scores.Count -gt 0) { [math]::Round((($scores | Measure-Object -Average).Average * 100), 1) } else { $null } + avgTokens = if ($tokens.Count -gt 0) { [int](($tokens | Measure-Object -Average).Average) } else { $null } + avgTurns = if ($turns.Count -gt 0) { [math]::Round((($turns | Measure-Object -Average).Average), 1) } else { $null } + avgToolCalls = if ($toolCalls.Count -gt 0) { [math]::Round((($toolCalls | Measure-Object -Average).Average), 1) } else { $null } + avgWallTimeSec = if ($wallTimesMs.Count -gt 0) { [math]::Round(((($wallTimesMs | Measure-Object -Average).Average) / 1000), 1) } else { $null } + totalErrors = if ($errors.Count -gt 0) { [int](($errors | Measure-Object -Sum).Sum) } else { 0 } + totalSkillActivations = if ($skillActivations.Count -gt 0) { [int](($skillActivations | Measure-Object -Sum).Sum) } else { 0 } + toolCallBreakdownTotals = $toolBreakdownTotals + skillActivationBreakdownTotals = $skillBreakdownTotals + totalFailedGraders = $totalFailedGraders + totalTimeoutGraders = $totalTimeoutGraders + failedGraderBreakdown = $failedGraderBreakdown + timeoutGraderBreakdown = $timeoutGraderBreakdown + stimuli = $stimuli + } +} + +function Invoke-ExternalProcessCaptured { + param( + [Parameter(Mandatory = $true)] + [string]$FilePath, + + [Parameter(Mandatory = $false)] + [string[]]$ArgumentList = @() + ) + + try { + $resolvedPath = $FilePath + $resolvedCommand = Get-Command -Name $FilePath -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($resolvedCommand -and -not [string]::IsNullOrWhiteSpace($resolvedCommand.Source)) { + $resolvedPath = $resolvedCommand.Source + } + + $effectiveFilePath = $resolvedPath + $effectiveArgs = New-Object System.Collections.Generic.List[string] + foreach ($arg in $ArgumentList) { + [void]$effectiveArgs.Add([string]$arg) + } + + if ($resolvedPath -and $resolvedPath.EndsWith('.ps1', [System.StringComparison]::OrdinalIgnoreCase)) { + $pwshCommand = Get-Command -Name pwsh -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $pwshCommand) { + $pwshCommand = Get-Command -Name powershell -ErrorAction SilentlyContinue | Select-Object -First 1 + } + + if (-not $pwshCommand -or [string]::IsNullOrWhiteSpace($pwshCommand.Source)) { + throw "PowerShell executable not found while attempting to launch script: $resolvedPath" + } + + $effectiveFilePath = $pwshCommand.Source + $scriptArgs = New-Object System.Collections.Generic.List[string] + [void]$scriptArgs.Add('-NoProfile') + [void]$scriptArgs.Add('-File') + [void]$scriptArgs.Add($resolvedPath) + foreach ($arg in $effectiveArgs) { + [void]$scriptArgs.Add($arg) + } + $effectiveArgs = $scriptArgs + } + + $psi = [System.Diagnostics.ProcessStartInfo]::new() + $psi.FileName = $effectiveFilePath + $psi.UseShellExecute = $false + $psi.RedirectStandardOutput = $true + $psi.RedirectStandardError = $true + $psi.CreateNoWindow = $true + + foreach ($arg in $effectiveArgs) { + [void]$psi.ArgumentList.Add([string]$arg) + } + + $process = [System.Diagnostics.Process]::new() + $process.StartInfo = $psi + + [void]$process.Start() + $stdoutText = $process.StandardOutput.ReadToEnd() + $stderrText = $process.StandardError.ReadToEnd() + $process.WaitForExit() + + $combinedOutput = $stdoutText + if (-not [string]::IsNullOrWhiteSpace($stderrText)) { + if (-not [string]::IsNullOrWhiteSpace($combinedOutput)) { + $combinedOutput += "`n" + } + $combinedOutput += $stderrText + } + + return [pscustomobject]@{ + ExitCode = $process.ExitCode + Output = $combinedOutput.Trim() + Stderr = $stderrText.Trim() + } + } + catch { + return [pscustomobject]@{ + ExitCode = -1 + Output = "" + Stderr = $_.Exception.Message + } + } +} + +function Invoke-CopilotNarrative { + param( + [Parameter(Mandatory = $true)] + [string]$Prompt, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10)] + [int]$MaxAttempts = 2, + + [Parameter(Mandatory = $false)] + [bool]$EnableNodeLoaderFallback = $true + ) + + $copilotCommand = Get-Command copilot -ErrorAction SilentlyContinue + if (-not $copilotCommand) { + return $null + } + + $copilotSource = $copilotCommand.Source + $copilotDir = Split-Path -Path $copilotSource -Parent + + $npmLoaderPath = $null + $loaderCandidates = @( + (Join-Path $copilotDir "node_modules\@github\copilot\npm-loader.js"), + "q:\.tools\.npm-global\node_modules\@github\copilot\npm-loader.js" + ) + + try { + $npmGlobalRoot = (& npm root -g 2>$null | Select-Object -First 1) + if (-not [string]::IsNullOrWhiteSpace($npmGlobalRoot)) { + $loaderCandidates += (Join-Path $npmGlobalRoot.Trim() "@github\copilot\npm-loader.js") + } + } + catch { + # Ignore npm-root discovery failures and continue with known candidates. + } + + foreach ($candidate in $loaderCandidates) { + if (-not [string]::IsNullOrWhiteSpace($candidate) -and (Test-Path $candidate)) { + $npmLoaderPath = $candidate + break + } + } + + $baseArgs = @("--silent", "--allow-all-tools", "--allow-all-paths", "-p", $Prompt) + + $attempts = @() + + $npxInvoker = $null + $npxCandidates = @("npx.ps1", "npx.cmd", "npx") + foreach ($candidate in $npxCandidates) { + $npxCommand = Get-Command -Name $candidate -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($npxCommand -and -not [string]::IsNullOrWhiteSpace($npxCommand.Source)) { + $npxInvoker = $npxCommand.Source + break + } + } + + if (-not $npxInvoker) { + $npxInvoker = "npx" + } + + # First choice: documented npx invocation. + $attempts += [pscustomobject]@{ + Name = "npx" + Cmd = $npxInvoker + Args = @("@github/copilot") + $baseArgs + } + + # Final fallback: use discovered copilot command path (prefer .cmd over .ps1). + $fallbackInvoker = $copilotSource + if ($fallbackInvoker -and $fallbackInvoker.EndsWith('.ps1', [System.StringComparison]::OrdinalIgnoreCase)) { + $cmdCandidate = [System.IO.Path]::ChangeExtension($fallbackInvoker, '.cmd') + if (Test-Path $cmdCandidate) { + $fallbackInvoker = $cmdCandidate + } + } + + $attempts += [pscustomobject]@{ + Name = "copilot-wrapper" + Cmd = $fallbackInvoker + Args = $baseArgs + } + + # Final fallback: npm-loader.js via node (undocumented implementation detail). + if ($EnableNodeLoaderFallback -and (Test-Path $npmLoaderPath)) { + $attempts += [pscustomobject]@{ + Name = "node-loader" + Cmd = "node" + Args = @($npmLoaderPath) + $baseArgs + } + } + + foreach ($attempt in $attempts) { + for ($i = 1; $i -le $MaxAttempts; $i++) { + Write-Verbose "Copilot narrative attempt '$($attempt.Name)' try $i/$MaxAttempts" + $execution = Invoke-ExternalProcessCaptured -FilePath $attempt.Cmd -ArgumentList @($attempt.Args) + + if ($execution.ExitCode -eq 0) { + Write-Verbose "Copilot narrative succeeded via '$($attempt.Name)'" + return $execution.Output + } + + if (-not [string]::IsNullOrWhiteSpace($execution.Stderr)) { + Write-Verbose "Copilot narrative '$($attempt.Name)' failed: $($execution.Stderr)" + } + else { + Write-Verbose "Copilot narrative '$($attempt.Name)' exited with code $($execution.ExitCode)" + } + + if ($i -lt $MaxAttempts) { + Start-Sleep -Milliseconds (250 * $i) + } + } + } + + return $null +} + +function Resolve-ExperimentOutputDir { + param( + [Parameter(Mandatory = $true)] + [string]$SkillDirPath + ) + + if (-not (Test-Path $SkillDirPath)) { + return $null + } + + $directVariantDirs = Get-ChildItem -Path $SkillDirPath -Directory -ErrorAction SilentlyContinue | Where-Object { + Test-Path (Join-Path $_.FullName "results.jsonl") + } + + if ($directVariantDirs.Count -ge 2) { + return $SkillDirPath + } + + $candidateRuns = Get-ChildItem -Path $SkillDirPath -Directory -ErrorAction SilentlyContinue | Sort-Object Name -Descending + foreach ($candidate in $candidateRuns) { + $variantDirs = Get-ChildItem -Path $candidate.FullName -Directory -ErrorAction SilentlyContinue | Where-Object { + Test-Path (Join-Path $_.FullName "results.jsonl") + } + if ($variantDirs.Count -ge 2) { + return $candidate.FullName + } + } + + return $null +} + +function Get-LatestResultsDirectory { + param( + [Parameter(Mandatory = $true)] + [string]$RootPath + ) + + if (-not (Test-Path $RootPath)) { + return $null + } + + $dirs = Get-ChildItem -Path $RootPath -Directory -ErrorAction SilentlyContinue | Sort-Object Name -Descending + if ($dirs.Count -eq 0) { + return $null + } + + return $dirs[0].FullName +} + +function Generate-NarrativesOnly { + param( + [Parameter(Mandatory = $true)] + [string]$AnalysisRoot, + + [Parameter(Mandatory = $true)] + [string]$SkillNamePattern, + + [Parameter(Mandatory = $false)] + [int]$NarrativeMaxAttempts = 2, + + [Parameter(Mandatory = $false)] + [bool]$EnableNodeLoaderFallback = $true + ) + + Write-Information "" + Write-Information "Analysis-Only Mode" + Write-Information "Source Results Directory: $AnalysisRoot" + + $skillDirs = Get-ChildItem -Path $AnalysisRoot -Directory -Filter $SkillNamePattern -ErrorAction SilentlyContinue | Sort-Object Name + if ($skillDirs.Count -eq 0) { + Write-Warning "No skill result directories found in $AnalysisRoot matching pattern: $SkillNamePattern" + return 1 + } + + $generated = 0 + $failed = 0 + + foreach ($skillDir in $skillDirs) { + $experimentDir = Resolve-ExperimentOutputDir -SkillDirPath $skillDir.FullName + if (-not $experimentDir) { + Write-Host " ✗ No valid experiment output found for $($skillDir.Name)" -ForegroundColor Yellow + $failed++ + continue + } + + $ok = Generate-SkillNarratives -SkillName $skillDir.Name -ExperimentOutputDir $experimentDir -SkillResultsDir $skillDir.FullName -NarrativeMaxAttempts $NarrativeMaxAttempts -EnableNodeLoaderFallback $EnableNodeLoaderFallback + if ($ok) { + $generated++ + Write-Host " ✓ Narratives generated for $($skillDir.Name)" -ForegroundColor Green + } + else { + $failed++ + Write-Host " ✗ Narrative generation failed for $($skillDir.Name)" -ForegroundColor Yellow + continue + } + } + + Write-Host "Narrative Summary: generated=$generated failed=$failed" -ForegroundColor Cyan + if ($failed -gt 0) { + return 1 + } + return 0 +} + +function Normalize-NarrativeText { + param( + [Parameter(Mandatory = $true)] + [string]$Text + ) + + # Keep markdown output plain ASCII to avoid rendering/encoding mismatches. + $normalized = $Text + $normalized = $normalized -replace '[^\x09\x0A\x0D\x20-\x7E]', '' + + return $normalized +} + +function Convert-ToolBreakdownToText { + param( + [Parameter(Mandatory = $false)] + $Breakdown + ) + + if (-not $Breakdown) { + return "none" + } + + $parts = @() + if ($Breakdown -is [System.Collections.IDictionary]) { + foreach ($key in $Breakdown.Keys) { + $parts += "${key}: $($Breakdown[$key])" + } + } + else { + $Breakdown.psobject.Properties | ForEach-Object { + $parts += "$($_.Name): $($_.Value)" + } + } + + if ($parts.Count -eq 0) { + return "none" + } + + return ($parts -join ", ") +} + +function Get-PromptFocusSummary { + param( + [Parameter(Mandatory = $false)] + [string]$Prompt + ) + + if ([string]::IsNullOrWhiteSpace($Prompt)) { + return "Prompt text not available in digest." + } + + $singleLine = ($Prompt -replace "`r?`n", " ").Trim() + $maxLen = 260 + if ($singleLine.Length -le $maxLen) { + return $singleLine + } + + return $singleLine.Substring(0, $maxLen).Trim() + "..." +} + +function Format-ElapsedDuration { + param( + [Parameter(Mandatory = $true)] + [double]$Seconds + ) + + $duration = [TimeSpan]::FromSeconds($Seconds) + return "{0}h {1}m {2:F1}s" -f [int]$duration.TotalHours, $duration.Minutes, ($duration.Seconds + ($duration.Milliseconds / 1000)) +} + +function New-VariantNarrativeFromDigest { + param( + [Parameter(Mandatory = $true)] + $Digest + ) + + $sb = New-Object System.Text.StringBuilder + $failedGraderNamesText = if ($Digest.failedGraderBreakdown -and $Digest.failedGraderBreakdown.Count -gt 0) { ($Digest.failedGraderBreakdown.Keys | Sort-Object) -join ", " } else { "none" } + $timeoutGraderNamesText = if ($Digest.timeoutGraderBreakdown -and $Digest.timeoutGraderBreakdown.Count -gt 0) { ($Digest.timeoutGraderBreakdown.Keys | Sort-Object) -join ", " } else { "none" } + + [void]$sb.AppendLine("# Execution Summary") + [void]$sb.AppendLine() + [void]$sb.AppendLine("Variant $($Digest.variant) in eval $($Digest.eval) with model $($Digest.model) ran $($Digest.trialCount) trials.") + [void]$sb.AppendLine("Average metrics for this variant: score $($Digest.avgScorePct)%, tokens $($Digest.avgTokens), turns $($Digest.avgTurns), tool calls $($Digest.avgToolCalls), wall time $($Digest.avgWallTimeSec)s.") + [void]$sb.AppendLine("Run totals for this variant: errors $($Digest.totalErrors), skill activations $($Digest.totalSkillActivations), tool usage $(Convert-ToolBreakdownToText -Breakdown $Digest.toolCallBreakdownTotals).") + [void]$sb.AppendLine("Grader outcomes across this variant: failed graders $($Digest.totalFailedGraders), timeout-related grader failures $($Digest.totalTimeoutGraders).") + [void]$sb.AppendLine("Failed grader names: $failedGraderNamesText. Timeout-related grader names: $timeoutGraderNamesText.") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Stimulus Narratives") + [void]$sb.AppendLine() + + foreach ($stimulus in $Digest.stimuli) { + [void]$sb.AppendLine("### Stimulus: $($stimulus.name)") + [void]$sb.AppendLine("- Prompt Focus: $(Get-PromptFocusSummary -Prompt $stimulus.prompt)") + [void]$sb.AppendLine("- Actions Taken: turns $($Digest.avgTurns) average for this variant; tool calls for this stimulus were $(Convert-ToolBreakdownToText -Breakdown $stimulus.toolCallBreakdown).") + + $graderParts = @() + foreach ($grader in $stimulus.graderOutcomes) { + $graderStatus = if ($grader.passed) { "passed" } else { "failed" } + $graderParts += "$($grader.name): $graderStatus ($($grader.score))" + } + $graderText = if ($graderParts.Count -gt 0) { $graderParts -join "; " } else { "none" } + + $resultStatus = if ($stimulus.passed) { "passed" } else { "failed" } + [void]$sb.AppendLine("- Result: stimulus $resultStatus with score $($stimulus.scorePct)% in $($stimulus.durationSec)s; graders: $graderText.") + + $failedGraders = @($stimulus.graderOutcomes | Where-Object { -not $_.passed }) + $timeoutGraders = @($failedGraders | Where-Object { $_.isTimeout }) + $nonTimeoutGraders = @($failedGraders | Where-Object { -not $_.isTimeout }) + + if ($failedGraders.Count -eq 0) { + [void]$sb.AppendLine("- Failed graders: none.") + } + else { + $nonTimeoutText = if ($nonTimeoutGraders.Count -gt 0) { ($nonTimeoutGraders | ForEach-Object { $_.name } | Sort-Object -Unique) -join ", " } else { "none" } + $timeoutText = if ($timeoutGraders.Count -gt 0) { ($timeoutGraders | ForEach-Object { $_.name } | Sort-Object -Unique) -join ", " } else { "none" } + [void]$sb.AppendLine("- Failed graders: non-timeout = $nonTimeoutText; timeout-related = $timeoutText.") + if ($timeoutGraders.Count -gt 0) { + [void]$sb.AppendLine("- Timeout note: one or more grader failures indicate timeout behavior; consider checking whether the stimulus timeout budget is too low for this scenario.") + } + } + [void]$sb.AppendLine() + } + + [void]$sb.AppendLine("# Tool and Turn Behavior") + [void]$sb.AppendLine() + [void]$sb.AppendLine("This variant averaged $($Digest.avgTurns) turns and $($Digest.avgToolCalls) tool calls per trial.") + [void]$sb.AppendLine("Aggregate tool breakdown for this variant: $(Convert-ToolBreakdownToText -Breakdown $Digest.toolCallBreakdownTotals).") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Grading Outcome") + [void]$sb.AppendLine() + [void]$sb.AppendLine("Trials run: $($Digest.trialCount). Average score for this variant: $($Digest.avgScorePct)%.") + [void]$sb.AppendLine("Total errors recorded: $($Digest.totalErrors).") + [void]$sb.AppendLine("Failed grader count: $($Digest.totalFailedGraders). Timeout-related grader failures: $($Digest.totalTimeoutGraders).") + [void]$sb.AppendLine("Failed grader names: $failedGraderNamesText. Timeout-related grader names: $timeoutGraderNamesText.") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Notable Strengths") + [void]$sb.AppendLine() + [void]$sb.AppendLine("- Completed $($Digest.trialCount) trials with total errors $($Digest.totalErrors).") + [void]$sb.AppendLine("- Produced runnable code artifacts validated during each trial.") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Notable Weaknesses") + [void]$sb.AppendLine() + if ([int]$Digest.totalFailedGraders -gt 0) { + [void]$sb.AppendLine("- Failed graders were recorded: $(Convert-ToolBreakdownToText -Breakdown $Digest.failedGraderBreakdown).") + } + else { + [void]$sb.AppendLine("- No failed graders were recorded in this variant.") + } + + if ([int]$Digest.totalTimeoutGraders -gt 0) { + [void]$sb.AppendLine("- Timeout-related grader failures were recorded: $(Convert-ToolBreakdownToText -Breakdown $Digest.timeoutGraderBreakdown). This often points to an undersized stimulus timeout budget rather than a logic error.") + } + + return $sb.ToString().Trim() +} + +function Get-PassRatePct { + param( + [Parameter(Mandatory = $true)] + $Digest + ) + + if (-not $Digest.stimuli -or $Digest.stimuli.Count -eq 0) { + return 0.0 + } + + $passedCount = @($Digest.stimuli | Where-Object { $_.passed }).Count + return [math]::Round((100.0 * $passedCount / $Digest.stimuli.Count), 1) +} + +function Get-OverallEffectLabel { + param( + [Parameter(Mandatory = $true)] + [double]$ScoreDelta, + + [Parameter(Mandatory = $true)] + [double]$PassRateDelta, + + [Parameter(Mandatory = $true)] + [double]$TokenDeltaPct, + + [Parameter(Mandatory = $true)] + [double]$WallTimeDeltaPct + ) + + $qualityImproved = ($ScoreDelta -ge 1.0) -or ($PassRateDelta -gt 0) + $qualityRegressed = ($ScoreDelta -le -1.0) -or ($PassRateDelta -lt 0) + $efficiencyImproved = ($TokenDeltaPct -le -5.0) -or ($WallTimeDeltaPct -le -5.0) + $efficiencyRegressed = ($TokenDeltaPct -ge 5.0) -or ($WallTimeDeltaPct -ge 5.0) + + if ($qualityImproved -and -not $qualityRegressed -and -not $efficiencyRegressed) { + return "Improved" + } + if ($qualityRegressed -and -not $qualityImproved -and -not $efficiencyImproved) { + return "Regressed" + } + if (($qualityImproved -and $efficiencyRegressed) -or ($qualityRegressed -and $efficiencyImproved)) { + return "Mixed" + } + + return "Neutral" +} + +function New-ComparisonNarrativeFromDigests { + param( + [Parameter(Mandatory = $true)] + [string]$SkillName, + + [Parameter(Mandatory = $true)] + [string]$BaselineVariant, + + [Parameter(Mandatory = $true)] + [string]$SkillVariant, + + [Parameter(Mandatory = $true)] + $BaselineDigest, + + [Parameter(Mandatory = $true)] + $SkillDigest + ) + + $baselineScore = [double]$BaselineDigest.avgScorePct + $skillScore = [double]$SkillDigest.avgScorePct + $scoreDelta = [math]::Round(($skillScore - $baselineScore), 1) + + $baselinePassRate = Get-PassRatePct -Digest $BaselineDigest + $skillPassRate = Get-PassRatePct -Digest $SkillDigest + $passRateDelta = [math]::Round(($skillPassRate - $baselinePassRate), 1) + + $baselineTokens = [double]$BaselineDigest.avgTokens + $skillTokens = [double]$SkillDigest.avgTokens + $tokenDeltaPct = if ($baselineTokens -gt 0) { [math]::Round((($skillTokens - $baselineTokens) / $baselineTokens) * 100.0, 1) } else { 0.0 } + + $baselineWall = [double]$BaselineDigest.avgWallTimeSec + $skillWall = [double]$SkillDigest.avgWallTimeSec + $wallTimeDeltaPct = if ($baselineWall -gt 0) { [math]::Round((($skillWall - $baselineWall) / $baselineWall) * 100.0, 1) } else { 0.0 } + + $overallEffect = Get-OverallEffectLabel -ScoreDelta $scoreDelta -PassRateDelta $passRateDelta -TokenDeltaPct $tokenDeltaPct -WallTimeDeltaPct $wallTimeDeltaPct + + $sb = New-Object System.Text.StringBuilder + + [void]$sb.AppendLine("# Overall Effect") + [void]$sb.AppendLine() + [void]$sb.AppendLine("$overallEffect") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Evidence From Process") + [void]$sb.AppendLine() + [void]$sb.AppendLine("- Turns: $BaselineVariant = $($BaselineDigest.avgTurns), $SkillVariant = $($SkillDigest.avgTurns) (delta $([math]::Round(([double]$SkillDigest.avgTurns - [double]$BaselineDigest.avgTurns), 1))).") + [void]$sb.AppendLine("- Tool calls: $BaselineVariant = $($BaselineDigest.avgToolCalls), $SkillVariant = $($SkillDigest.avgToolCalls) (delta $([math]::Round(([double]$SkillDigest.avgToolCalls - [double]$BaselineDigest.avgToolCalls), 1))).") + [void]$sb.AppendLine("- Skill activations: $BaselineVariant = $($BaselineDigest.totalSkillActivations), $SkillVariant = $($SkillDigest.totalSkillActivations).") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Evidence From Outcomes") + [void]$sb.AppendLine() + [void]$sb.AppendLine("- Average score: $BaselineVariant = $baselineScore%, $SkillVariant = $skillScore% (delta $scoreDelta points).") + [void]$sb.AppendLine("- Stimulus pass rate: $BaselineVariant = $baselinePassRate%, $SkillVariant = $skillPassRate% (delta $passRateDelta points).") + [void]$sb.AppendLine("- Total errors: $BaselineVariant = $($BaselineDigest.totalErrors), $SkillVariant = $($SkillDigest.totalErrors).") + [void]$sb.AppendLine("- Failed graders: $BaselineVariant = $($BaselineDigest.totalFailedGraders), $SkillVariant = $($SkillDigest.totalFailedGraders) (delta $([int]$SkillDigest.totalFailedGraders - [int]$BaselineDigest.totalFailedGraders)).") + [void]$sb.AppendLine("- Timeout-related grader failures: $BaselineVariant = $($BaselineDigest.totalTimeoutGraders), $SkillVariant = $($SkillDigest.totalTimeoutGraders) (delta $([int]$SkillDigest.totalTimeoutGraders - [int]$BaselineDigest.totalTimeoutGraders)).") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Efficiency Tradeoffs") + [void]$sb.AppendLine() + [void]$sb.AppendLine("- Tokens: $BaselineVariant = $([int]$baselineTokens), $SkillVariant = $([int]$skillTokens) (delta $tokenDeltaPct%).") + [void]$sb.AppendLine("- Wall time: $BaselineVariant = $baselineWall s, $SkillVariant = $skillWall s (delta $wallTimeDeltaPct%).") + [void]$sb.AppendLine() + + [void]$sb.AppendLine("# Final Assessment") + [void]$sb.AppendLine() + [void]$sb.AppendLine("For $SkillName, the deterministic comparison indicates $overallEffect.") + if ([int]$SkillDigest.totalTimeoutGraders -gt [int]$BaselineDigest.totalTimeoutGraders) { + [void]$sb.AppendLine("Timeout-related grader failures increased in $SkillVariant; this may indicate the stimulus timeout threshold is too low for the executed workflow and should be reviewed.") + } + [void]$sb.AppendLine("This assessment is computed directly from trajectory digests to avoid narrative drift or missing-context artifacts.") + + return $sb.ToString().Trim() +} + +function Generate-SkillNarratives { + param( + [Parameter(Mandatory = $true)] + [string]$SkillName, + + [Parameter(Mandatory = $true)] + [string]$ExperimentOutputDir, + + [Parameter(Mandatory = $true)] + [string]$SkillResultsDir, + + [Parameter(Mandatory = $false)] + [ValidateRange(1, 10)] + [int]$NarrativeMaxAttempts = 2, + + [Parameter(Mandatory = $false)] + [bool]$EnableNodeLoaderFallback = $true + ) + + if (-not (Test-Path $ExperimentOutputDir)) { + return $false + } + + New-Item -ItemType Directory -Path $SkillResultsDir -Force | Out-Null + + $variantDirs = Get-ChildItem -Path $ExperimentOutputDir -Directory | Where-Object { + Test-Path (Join-Path $_.FullName "results.jsonl") + } + + if ($variantDirs.Count -lt 2) { + Write-Warning "Expected at least 2 variants in $ExperimentOutputDir for $SkillName. Found $($variantDirs.Count)." + return $false + } + + $digests = @{} + $narratives = @{} + + foreach ($variantDir in $variantDirs) { + $resultsJsonlPath = Join-Path $variantDir.FullName "results.jsonl" + $digest = Get-VariantDigest -ResultsJsonlPath $resultsJsonlPath + if (-not $digest) { + continue + } + + $variantName = [string]$digest.variant + $digests[$variantName] = $digest + + $digestPath = Join-Path $SkillResultsDir ("trajectory-digest-" + $variantName + ".json") + $digest | ConvertTo-Json -Depth 12 | Set-Content -Path $digestPath -Encoding UTF8 + + $narrative = New-VariantNarrativeFromDigest -Digest $digest + + $narrative = Normalize-NarrativeText -Text $narrative + + $narrativePath = Join-Path $SkillResultsDir ("narrative-" + $variantName + ".md") + $narrative | Set-Content -Path $narrativePath -Encoding UTF8BOM + $narratives[$variantName] = $narrative + } + + if ($digests.Count -lt 2 -or $narratives.Count -lt 2) { + Write-Warning "Could not build both variant narratives for $SkillName" + return $false + } + + $variantNames = @($digests.Keys | Sort-Object) + $baselineVariant = $variantNames | Where-Object { $_ -match 'baseline' } | Select-Object -First 1 + $skillVariant = $variantNames | Where-Object { $_ -match 'skill' } | Select-Object -First 1 + + if (-not $baselineVariant) { + $baselineVariant = $variantNames[0] + } + if (-not $skillVariant) { + $skillVariant = $variantNames | Where-Object { $_ -ne $baselineVariant } | Select-Object -First 1 + } + + if (-not $skillVariant) { + Write-Warning "Unable to identify comparison pair for $SkillName" + return $false + } + + $baselineDigest = $digests[$baselineVariant] + $skillDigest = $digests[$skillVariant] + + $comparison = New-ComparisonNarrativeFromDigests -SkillName $SkillName -BaselineVariant $baselineVariant -SkillVariant $skillVariant -BaselineDigest $baselineDigest -SkillDigest $skillDigest + + $comparison = Normalize-NarrativeText -Text $comparison + + $comparisonPath = Join-Path $SkillResultsDir "narrative-comparison.md" + $comparison | Set-Content -Path $comparisonPath -Encoding UTF8BOM + + return $true +} + +$copilotCmd = Get-Command copilot -ErrorAction SilentlyContinue +if ($copilotCmd) { + Write-Information "Copilot CLI: $($copilotCmd.Source)" +} +else { + Write-Warning "copilot CLI not found. Narrative generation will be skipped." +} + +if ($AnalysisOnly) { + $analysisRoot = $ExperimentOutputDir + if ([string]::IsNullOrWhiteSpace($analysisRoot)) { + $analysisRoot = Get-LatestResultsDirectory -RootPath $ResultsRoot + } + + if (-not $analysisRoot) { + Write-Error "Analysis-only mode could not resolve a results directory. Provide -ExperimentOutputDir explicitly." + exit 1 + } + + $analysisExitCode = Generate-NarrativesOnly -AnalysisRoot $analysisRoot -SkillNamePattern $SkillPattern -NarrativeMaxAttempts $CopilotMaxAttempts -EnableNodeLoaderFallback (-not $NoNodeLoaderFallback) + exit $analysisExitCode +} + +# Validate vally is available +$vallyCmd = Get-Command vally -ErrorAction SilentlyContinue +if (-not $vallyCmd) { + Write-Error "vally command not found. Please install Vally to run experiments." + exit 1 +} + +Write-Information "Vally: $($vallyCmd.Source)" + +# Find all matching skill scenarios with experiments +$skillDirs = Get-ChildItem -Path $ScenariosRoot -Directory -Filter $SkillPattern | +Where-Object { Test-Path (Join-Path $_.FullName "vally" "skill_effectiveness_experiment.yaml") } | +Sort-Object Name + +Write-Information "Found $($skillDirs.Count) skills with experiments matching pattern: $SkillPattern" + +if ($skillDirs.Count -eq 0) { + Write-Warning "No skills found matching pattern: $SkillPattern" + exit 0 +} + +if ($DryRun) { + Write-Information "" + Write-Information "DRY RUN - Would execute the following experiments:" + foreach ($skillDir in $skillDirs) { + Write-Information " - $($skillDir.Name): $(Join-Path $skillDir.FullName 'vally' 'skill_effectiveness_experiment.yaml')" + } + exit 0 +} + +# Create results directory with timestamp +$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fffZ" +$experimentResultsDir = Join-Path $ResultsRoot $timestamp +New-Item -ItemType Directory -Path $experimentResultsDir -Force | Out-Null + +Write-Information "" +Write-Information "Results Directory: $experimentResultsDir" +Write-Information "Parallel Workers: $Workers" +Write-Information "" + +# Track results and timing +$results = @() +$completed = 0 +$failed = 0 +$totalSkills = $skillDirs.Count +$overallStartTime = Get-Date +$skillTimings = @() # Track timings for ETA calculation + +# Execute experiments +$scriptBlock = { + param($SkillDir, $ExperimentResultsDir, $ResultsRoot, $SkillIndex, $TotalSkills) + + $skillName = $SkillDir.Name + $vallyDir = Join-Path $SkillDir.FullName "vally" + try { + Write-Host "[$SkillIndex/$TotalSkills] ▶ Starting: $skillName" + + $startTime = Get-Date + $skillOutputRoot = Join-Path $ExperimentResultsDir $skillName + + # Run the experiment + # Note: vally experiment run expects to be run from the scenario directory + Push-Location $vallyDir + try { + $output = & vally experiment run skill_effectiveness_experiment.yaml --output-dir $skillOutputRoot 2>&1 + $exitCode = $LASTEXITCODE + } + finally { + Pop-Location + } + + $duration = (Get-Date) - $startTime + + # Check for success by looking at output and exit code + # Vally returns 0 on success, but also check for "passed" keyword in output + $outputString = $output -join "`n" + + # Strip ANSI color codes from output to allow proper regex matching + # vally uses ANSI codes for colored output which blocks metric extraction + $outputString = $outputString -replace "`e\[[0-9;]*m", '' + + # Parse experiment output directory locally so this works in Start-Job scope. + $experimentOutputDir = $null + $localOutputMatches = [regex]::Matches($outputString, 'Output:\s*(.+)') + if ($localOutputMatches.Count -gt 0) { + for ($m = $localOutputMatches.Count - 1; $m -ge 0; $m--) { + $candidatePath = $localOutputMatches[$m].Groups[1].Value.Trim() + if (Test-Path $candidatePath) { + $experimentOutputDir = $candidatePath + break + } + } + + if (-not $experimentOutputDir) { + $experimentOutputDir = $localOutputMatches[$localOutputMatches.Count - 1].Groups[1].Value.Trim() + } + } + + # Prefer deterministic per-variant metrics from JSONL results over parsing console text. + $baselineScore = $null + $skillScore = $null + $delta = $null + $baselineTokens = $null + $skillTokens = $null + $baselineTurns = $null + $skillTurns = $null + $baselineErrors = 0 + $skillErrors = 0 + $baselineFailedGraders = 0 + $skillFailedGraders = 0 + $baselineTimeoutGraders = 0 + $skillTimeoutGraders = 0 + $baselineEvalStatus = "UNKNOWN" + $skillEvalStatus = "UNKNOWN" + $succeeded = $false + + $variantStats = @{} + if ($experimentOutputDir -and (Test-Path $experimentOutputDir)) { + $variantDirs = Get-ChildItem -Path $experimentOutputDir -Directory -ErrorAction SilentlyContinue + foreach ($variantDir in $variantDirs) { + $resultsFile = Join-Path $variantDir.FullName "results.jsonl" + if (-not (Test-Path $resultsFile)) { + continue + } + + $scores = @() + $tokens = @() + $turns = @() + $errors = 0 + $failedGraders = 0 + $timeoutGraders = 0 + $trialCount = 0 + $passedCount = 0 + + foreach ($line in (Get-Content -Path $resultsFile -ErrorAction SilentlyContinue)) { + if ([string]::IsNullOrWhiteSpace($line)) { + continue + } + + try { + $record = $line | ConvertFrom-Json -ErrorAction Stop + } + catch { + continue + } + + if ($record.type -ne "trial-result") { + continue + } + + $trialCount++ + if ($record.gradeResult.passed) { + $passedCount++ + } + + if ($null -ne $record.gradeResult.score) { + $scores += ([double]$record.gradeResult.score * 100) + } + + if ($null -ne $record.trajectory.metrics.turnCount) { + $turns += [int]$record.trajectory.metrics.turnCount + } + + if ($null -ne $record.trajectory.metrics.tokenUsage.totalTokens) { + $tokens += [int]$record.trajectory.metrics.tokenUsage.totalTokens + } + + if ($null -ne $record.trajectory.metrics.errorCount) { + $errors += [int]$record.trajectory.metrics.errorCount + } + + if ($record.gradeResult -and $record.gradeResult.details) { + foreach ($detail in $record.gradeResult.details) { + if (-not [bool]$detail.passed) { + $failedGraders++ + $detailEvidence = if ($null -ne $detail.evidence) { [string]$detail.evidence } else { "" } + if ($detailEvidence -match '(?i)\btime(d)?\s*out\b|\btimeout\b') { + $timeoutGraders++ + } + } + } + } + } + + if ($trialCount -gt 0) { + $variantStats[$variantDir.Name] = @{ + AvgScore = if ($scores.Count -gt 0) { [double]($scores | Measure-Object -Average).Average } else { $null } + AvgTokens = if ($tokens.Count -gt 0) { [int](($tokens | Measure-Object -Average).Average) } else { $null } + AvgTurns = if ($turns.Count -gt 0) { [int](($turns | Measure-Object -Average).Average) } else { $null } + TotalErrors = $errors + FailedGraders = $failedGraders + TimeoutGraders = $timeoutGraders + TrialCount = $trialCount + PassedCount = $passedCount + } + } + } + } + + $baselineVariantName = $null + $skillVariantName = $null + if ($variantStats.Count -gt 0) { + $variantNames = @($variantStats.Keys | Sort-Object) + $baselineVariantName = $variantNames | Where-Object { $_ -match 'baseline' } | Select-Object -First 1 + $skillVariantName = $variantNames | Where-Object { $_ -match 'skill' } | Select-Object -First 1 + + if (-not $baselineVariantName) { + $baselineVariantName = $variantNames | Select-Object -First 1 + } + if (-not $skillVariantName) { + $skillVariantName = $variantNames | Where-Object { $_ -ne $baselineVariantName } | Select-Object -First 1 + } + } + + if ($baselineVariantName -and $variantStats.ContainsKey($baselineVariantName)) { + $baselineStats = $variantStats[$baselineVariantName] + $baselineScore = $baselineStats.AvgScore + $baselineTokens = $baselineStats.AvgTokens + $baselineTurns = $baselineStats.AvgTurns + $baselineErrors = $baselineStats.TotalErrors + $baselineFailedGraders = $baselineStats.FailedGraders + $baselineTimeoutGraders = $baselineStats.TimeoutGraders + if ($baselineStats.TrialCount -gt 0) { + $baselineEvalStatus = if ($baselineStats.PassedCount -eq $baselineStats.TrialCount) { "PASSED" } else { "FAILED" } + } + } + + if ($skillVariantName -and $variantStats.ContainsKey($skillVariantName)) { + $skillStats = $variantStats[$skillVariantName] + $skillScore = $skillStats.AvgScore + $skillTokens = $skillStats.AvgTokens + $skillTurns = $skillStats.AvgTurns + $skillErrors = $skillStats.TotalErrors + $skillFailedGraders = $skillStats.FailedGraders + $skillTimeoutGraders = $skillStats.TimeoutGraders + $succeeded = ($exitCode -eq 0) -and ($skillStats.TrialCount -gt 0) -and ($skillStats.PassedCount -eq $skillStats.TrialCount) + if ($skillStats.TrialCount -gt 0) { + $skillEvalStatus = if ($skillStats.PassedCount -eq $skillStats.TrialCount) { "PASSED" } else { "FAILED" } + } + } + else { + # Fallback for older outputs when per-variant JSONL is unavailable. + $succeeded = ($exitCode -eq 0) -and ($outputString -match "passed") + } + + if (($null -ne $baselineScore) -and ($null -ne $skillScore)) { + $delta = [double]$skillScore - [double]$baselineScore + } + + if ($succeeded) { + # Format impact indicator with efficiency metrics + $impactStr = "" + if ($delta -ne $null) { + $impactDirection = if ($delta -ge 0) { "↑" } else { "↓" } + $impactStr = " [baseline: $($baselineScore.ToString('F1'))% → skill: $($skillScore.ToString('F1'))% $impactDirection $($delta.ToString('+0.0;-0.0'))%]" + } + elseif ($skillScore) { + $impactStr = " [skill: $($skillScore.ToString('F1'))%]" + } + if ($skillEvalStatus -ne "UNKNOWN") { + $impactStr += " [skill eval: $skillEvalStatus]" + } + if ($skillFailedGraders -gt 0) { + $impactStr += " [failed graders: $skillFailedGraders" + if ($skillTimeoutGraders -gt 0) { + $impactStr += ", timeout-related: $skillTimeoutGraders" + } + $impactStr += "]" + } + + $durationText = "{0}h {1}m {2:F1}s" -f [int]$duration.TotalHours, $duration.Minutes, ($duration.Seconds + ($duration.Milliseconds / 1000)) + + Write-Host "[$SkillIndex/$TotalSkills] ✓ PASSED (run): $skillName ($durationText)$impactStr" -ForegroundColor Green + return @{ + Skill = $skillName + Status = "PASSED" + SkillEvalStatus = $skillEvalStatus + Duration = $duration.TotalSeconds + BaselineScore = $baselineScore + SkillScore = $skillScore + Delta = $delta + BaselineTokens = $baselineTokens + SkillTokens = $skillTokens + BaselineTurns = $baselineTurns + SkillTurns = $skillTurns + BaselineErrors = $baselineErrors + SkillErrors = $skillErrors + BaselineFailedGraders = $baselineFailedGraders + SkillFailedGraders = $skillFailedGraders + BaselineTimeoutGraders = $baselineTimeoutGraders + SkillTimeoutGraders = $skillTimeoutGraders + BaselineEvalStatus = $baselineEvalStatus + ExperimentDir = $experimentOutputDir + Output = $output + } + } + else { + Write-Host "[$SkillIndex/$TotalSkills] ✗ FAILED: $skillName (exit code: $exitCode)" -ForegroundColor Red + return @{ + Skill = $skillName + Status = "FAILED" + SkillEvalStatus = $skillEvalStatus + Duration = $duration.TotalSeconds + BaselineScore = $baselineScore + SkillScore = $skillScore + Delta = $delta + BaselineTokens = $baselineTokens + SkillTokens = $skillTokens + BaselineTurns = $baselineTurns + SkillTurns = $skillTurns + BaselineErrors = $baselineErrors + SkillErrors = $skillErrors + BaselineFailedGraders = $baselineFailedGraders + SkillFailedGraders = $skillFailedGraders + BaselineTimeoutGraders = $baselineTimeoutGraders + SkillTimeoutGraders = $skillTimeoutGraders + BaselineEvalStatus = $baselineEvalStatus + ExperimentDir = $experimentOutputDir + Output = $output + } + } + } + catch { + Write-Host "[$SkillIndex/$TotalSkills] ✗ Error running $skillName : $_" + return @{ + Skill = $skillName + Status = "ERROR" + Duration = 0 + Score = $null + Output = $_.Exception.Message + } + } +} + +# Run experiments with sequential or throttled-parallel execution +$skillIndex = 0 + +if ($Workers -eq 1) { + foreach ($skillDir in $skillDirs) { + $skillIndex++ + + $result = & $scriptBlock $skillDir $experimentResultsDir $ResultsRoot $skillIndex $totalSkills + $results += $result + $skillTimings += $result.Duration + + if ($result.Status -ne "PASSED") { + $failed++ + } + else { + $completed++ + } + + # Show ETA for sequential execution + if ($skillIndex -lt $totalSkills -and $skillTimings.Count -gt 0) { + $avgTime = ($skillTimings | Measure-Object -Average).Average + $remainingSkills = $totalSkills - $skillIndex + $estimatedSeconds = [int]($avgTime * $remainingSkills) + $estimatedTime = "{0:D2}:{1:D2}" -f [int]($estimatedSeconds / 60), [int]($estimatedSeconds % 60) + Write-Host " → Estimated time remaining: $estimatedTime" -ForegroundColor Cyan + } + } +} +else { + Write-Host "Launching experiments with worker limit: $Workers" -ForegroundColor Cyan + + $activeJobs = @() + $pendingSkills = [System.Collections.Queue]::new() + foreach ($skillDir in $skillDirs) { + [void]$pendingSkills.Enqueue($skillDir) + } + + function Start-NextExperimentJob { + param( + [Parameter(Mandatory = $true)] + [System.Collections.Queue]$Queue, + + [Parameter(Mandatory = $true)] + [int]$NextIndex, + + [Parameter(Mandatory = $true)] + [string]$ExpResultsDir, + + [Parameter(Mandatory = $true)] + [string]$ResRoot, + + [Parameter(Mandatory = $true)] + [int]$TotalCount, + + [Parameter(Mandatory = $true)] + [scriptblock]$RunnerScript + ) + + if ($Queue.Count -eq 0) { + return $null + } + + $nextSkillDir = $Queue.Dequeue() + $newJob = Start-Job -ScriptBlock $RunnerScript -ArgumentList $nextSkillDir, $ExpResultsDir, $ResRoot, $NextIndex, $TotalCount + + return @{ + Job = $newJob + Skill = $nextSkillDir.Name + SkillIndex = $NextIndex + } + } + + # Seed up to worker limit. + while ($activeJobs.Count -lt $Workers -and $pendingSkills.Count -gt 0) { + $skillIndex++ + $jobInfo = Start-NextExperimentJob -Queue $pendingSkills -NextIndex $skillIndex -ExpResultsDir $experimentResultsDir -ResRoot $ResultsRoot -TotalCount $totalSkills -RunnerScript $scriptBlock + if ($jobInfo) { + $activeJobs += $jobInfo + Write-Host " queued [$($jobInfo.SkillIndex)/$totalSkills] $($jobInfo.Skill) (active: $($activeJobs.Count)/$Workers)" -ForegroundColor DarkCyan + } + } + + # As jobs finish, consume output and backfill from queue. + while ($activeJobs.Count -gt 0) { + $jobObjects = @($activeJobs | ForEach-Object { $_.Job }) + $finishedJob = Wait-Job -Job $jobObjects -Any + if (-not $finishedJob) { + continue + } + + $jobInfo = $activeJobs | Where-Object { $_.Job.Id -eq $finishedJob.Id } | Select-Object -First 1 + $result = Receive-Job -Job $finishedJob -Wait -InformationAction SilentlyContinue + Remove-Job -Job $finishedJob + + $activeJobs = @($activeJobs | Where-Object { $_.Job.Id -ne $finishedJob.Id }) + + $results += $result + $skillTimings += $result.Duration + + if ($result.Status -ne "PASSED") { + $failed++ + } + else { + $completed++ + } + + $done = $completed + $failed + Write-Host " progress: $done/$totalSkills complete (active: $($activeJobs.Count), queued: $($pendingSkills.Count))" -ForegroundColor Cyan + + if ($pendingSkills.Count -gt 0) { + $skillIndex++ + $nextJobInfo = Start-NextExperimentJob -Queue $pendingSkills -NextIndex $skillIndex -ExpResultsDir $experimentResultsDir -ResRoot $ResultsRoot -TotalCount $totalSkills -RunnerScript $scriptBlock + if ($nextJobInfo) { + $activeJobs += $nextJobInfo + Write-Host " queued [$($nextJobInfo.SkillIndex)/$totalSkills] $($nextJobInfo.Skill) (active: $($activeJobs.Count)/$Workers)" -ForegroundColor DarkCyan + } + } + } +} + +Write-Information "" +Write-Information "Generating trajectory narratives and skill-impact comparisons..." + +$narrativeCount = 0 +$narrativeFailures = 0 + +foreach ($result in $results) { + if (-not $result.ExperimentDir) { + continue + } + + $skillNarrativeDir = Join-Path $experimentResultsDir $result.Skill + $generated = Generate-SkillNarratives -SkillName $result.Skill -ExperimentOutputDir $result.ExperimentDir -SkillResultsDir $skillNarrativeDir -NarrativeMaxAttempts $CopilotMaxAttempts -EnableNodeLoaderFallback (-not $NoNodeLoaderFallback) + + if ($generated) { + $narrativeCount++ + Write-Host " ✓ Narratives generated for $($result.Skill)" -ForegroundColor Green + } + else { + $narrativeFailures++ + Write-Host " ✗ Narrative generation skipped/failed for $($result.Skill)" -ForegroundColor Yellow + } +} + +if ($narrativeCount -gt 0 -or $narrativeFailures -gt 0) { + Write-Host "Narrative Summary: generated=$narrativeCount failed=$narrativeFailures" -ForegroundColor Cyan +} + +# Calculate overall timing +$overallDuration = (Get-Date) - $overallStartTime +$totalDurationStr = "{0}h {1}m {2}s" -f ` + [int]($overallDuration.TotalSeconds / 3600), ` + [int](($overallDuration.TotalSeconds % 3600) / 60), ` + [int]($overallDuration.TotalSeconds % 60) + +Write-Host "`n" + ("=" * 80) +Write-Host "Skill Effectiveness Experiments Summary" -ForegroundColor Green +Write-Host ("=" * 80) + +# Build results table with verdict and efficiency analysis +$resultsTable = @() +foreach ($result in $results) { + # Determine verdict based on pass/fail logic + $verdict = "" + $evalStatus = if ($result.SkillEvalStatus) { [string]$result.SkillEvalStatus } else { "UNKNOWN" } + $baselineEvalStatus = if ($result.BaselineEvalStatus) { [string]$result.BaselineEvalStatus } else { "UNKNOWN" } + $failureArm = "unknown" + + if ($result.Status -eq "FAILED") { + if ($baselineEvalStatus -eq "FAILED" -and $evalStatus -ne "FAILED") { + $failureArm = "baseline" + } + elseif ($evalStatus -eq "FAILED" -and $baselineEvalStatus -ne "FAILED") { + $failureArm = "skill" + } + elseif ($baselineEvalStatus -eq "FAILED" -and $evalStatus -eq "FAILED") { + $failureArm = "both" + } + else { + $failureArm = "run" + } + } + + if ($result.Status -eq "PASSED" -and $evalStatus -eq "PASSED") { + # Skill passed - check efficiency + if ($result.SkillTokens -and $result.BaselineTokens) { + $tokenDelta = [int](($result.SkillTokens - $result.BaselineTokens) / $result.BaselineTokens * 100) + if ($tokenDelta -lt -5) { + $verdict = "✓ Effective" + } + elseif ($tokenDelta -gt 5) { + $verdict = "✗ Inefficient" + } + else { + $verdict = "~ Neutral" + } + } + else { + $verdict = "✓ Passed" + } + } + elseif ($result.Status -eq "PASSED" -and $evalStatus -eq "FAILED") { + if (($result.SkillTimeoutGraders -as [int]) -gt 0) { + $verdict = "✗ Eval failed (timeout)" + } + else { + $verdict = "✗ Eval failed" + } + } + else { + $verdict = "✗ Failed" + } + + # Calculate efficiency deltas + $tokensDelta = "" + $turnsDelta = "" + + if ($result.SkillTokens -and $result.BaselineTokens) { + $pctChange = [int](($result.SkillTokens - $result.BaselineTokens) / $result.BaselineTokens * 100) + $dir = if ($pctChange -ge 0) { "↑" } else { "↓" } + $tokensDelta = "$dir $($pctChange.ToString('+0;-0'))%" + } + + if ($result.SkillTurns -and $result.BaselineTurns) { + $turnChange = $result.SkillTurns - $result.BaselineTurns + $dir = if ($turnChange -ge 0) { "↑" } else { "↓" } + $turnsDelta = "$dir $($turnChange.ToString('+0;-0'))" + } + + # Build status indicator + $statusIcon = if ($result.Status -eq "PASSED") { "RUN PASS" } else { "RUN FAIL" } + + # Build verdict display + $verdictDisplay = $verdict -replace "✓ ", "[GOOD] " -replace "✗ ", "[BAD] " -replace "~ ", "[NEUT] " + + $resultsTable += [PSCustomObject]@{ + Skill = $result.Skill + Status = $statusIcon + Eval = $evalStatus + 'Failure Arm' = if ($result.Status -eq "FAILED") { $failureArm } else { "none" } + Verdict = $verdictDisplay + 'Graders Failed' = if ($null -ne $result.SkillFailedGraders) { $result.SkillFailedGraders } else { "N/A" } + 'Graders Timeout' = if ($null -ne $result.SkillTimeoutGraders) { $result.SkillTimeoutGraders } else { "N/A" } + 'Tokens %' = if ($tokensDelta) { $tokensDelta } else { "N/A" } + 'Turns' = if ($turnsDelta) { $turnsDelta } else { "N/A" } + Errors = $result.SkillErrors + 'Duration' = Format-ElapsedDuration -Seconds $result.Duration + } +} + +# Display results table +$resultsTable | Format-Table -AutoSize + +Write-Host ("=" * 80) +Write-Host "Results Summary" -ForegroundColor Cyan +Write-Host (" Total Skills: $totalSkills") +Write-Host (" Passed: $completed") -ForegroundColor Green +if ($failed -gt 0) { + Write-Host (" Failed: $failed") -ForegroundColor Red +} +else { + Write-Host (" Failed: $failed") -ForegroundColor Green +} +Write-Host (" Total Duration: $totalDurationStr") -ForegroundColor Cyan +Write-Host (" Results Dir: $experimentResultsDir") -ForegroundColor Cyan +Write-Host ("=" * 80) +Write-Information "" + +if ($Compare) { + Write-Information "" + Write-Information "Generating comparison reports..." + + # Run compare-experiment.mjs for the experiment directory + $compareScript = Join-Path $PSScriptRoot "compare-experiment.mjs" + if (Test-Path $compareScript) { + & node $compareScript $experimentResultsDir + } + else { + Write-Warning "compare-experiment.mjs not found at: $compareScript" + } +} + +if ($failed -gt 0) { + exit 1 +} diff --git a/tests/scenarios/_shared/vally/azure-sdk-python-skill-evaluations.yaml b/tests/scenarios/_shared/vally/azure-sdk-python-skill-evaluations.yaml new file mode 100644 index 00000000..ec39cd44 --- /dev/null +++ b/tests/scenarios/_shared/vally/azure-sdk-python-skill-evaluations.yaml @@ -0,0 +1,43 @@ +name: azure-sdk-python-skill-evaluations + +evals: + - ../../agent-framework-azure-ai-py/vally/eval.yaml + - ../../azure-ai-contentsafety-py/vally/eval.yaml + - ../../azure-ai-contentunderstanding-py/vally/eval.yaml + - ../../azure-ai-language-conversations-py/vally/eval.yaml + - ../../azure-ai-ml-py/vally/eval.yaml + - ../../azure-ai-projects-py/vally/eval.yaml + - ../../azure-ai-textanalytics-py/vally/eval.yaml + - ../../azure-ai-transcription-py/vally/eval.yaml + - ../../azure-ai-translation-document-py/vally/eval.yaml + - ../../azure-ai-translation-text-py/vally/eval.yaml + - ../../azure-ai-vision-imageanalysis-py/vally/eval.yaml + - ../../azure-ai-voicelive-py/vally/eval.yaml + - ../../azure-appconfiguration-py/vally/eval.yaml + - ../../azure-containerregistry-py/vally/eval.yaml + - ../../azure-cosmos-db-py/vally/eval.yaml + - ../../azure-cosmos-py/vally/eval.yaml + - ../../azure-data-tables-py/vally/eval.yaml + - ../../azure-eventgrid-py/vally/eval.yaml + - ../../azure-eventhub-py/vally/eval.yaml + - ../../azure-identity-py/vally/eval.yaml + - ../../azure-keyvault-py/vally/eval.yaml + - ../../azure-messaging-webpubsubservice-py/vally/eval.yaml + - ../../azure-mgmt-apicenter-py/vally/eval.yaml + - ../../azure-mgmt-apimanagement-py/vally/eval.yaml + - ../../azure-mgmt-botservice-py/vally/eval.yaml + - ../../azure-mgmt-fabric-py/vally/eval.yaml + - ../../azure-monitor-ingestion-py/vally/eval.yaml + - ../../azure-monitor-opentelemetry-exporter-py/vally/eval.yaml + - ../../azure-monitor-opentelemetry-py/vally/eval.yaml + - ../../azure-monitor-query-py/vally/eval.yaml + - ../../azure-search-documents-py/vally/eval.yaml + - ../../azure-servicebus-py/vally/eval.yaml + - ../../azure-speech-to-text-rest-py/vally/eval.yaml + - ../../azure-storage-blob-py/vally/eval.yaml + - ../../azure-storage-file-datalake-py/vally/eval.yaml + - ../../azure-storage-file-share-py/vally/eval.yaml + - ../../azure-storage-queue-py/vally/eval.yaml + - ../../fastapi-router-py/vally/eval.yaml + - ../../m365-agents-py/vally/eval.yaml + - ../../pydantic-models-py/vally/eval.yaml diff --git a/tests/scenarios/_shared/vally/azure-sdk-rust-skill-experiment.yaml b/tests/scenarios/_shared/vally/azure-sdk-rust-skill-experiment.yaml index 3ed92cb2..a1ae8d18 100644 --- a/tests/scenarios/_shared/vally/azure-sdk-rust-skill-experiment.yaml +++ b/tests/scenarios/_shared/vally/azure-sdk-rust-skill-experiment.yaml @@ -7,6 +7,5 @@ evals: - ../../azure-keyvault-certificates-rust/vally/eval.yaml - ../../azure-keyvault-keys-rust/vally/eval.yaml - ../../azure-keyvault-secrets-rust/vally/eval.yaml - - ../../azure-servicebus-rust/vally/eval.yaml - ../../azure-storage-blob-rust/vally/eval.yaml - ../../azure-storage-queue-rust/vally/eval.yaml diff --git a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts index df07f2e8..786fbb03 100644 --- a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts +++ b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/index.ts @@ -424,9 +424,8 @@ class CargoBuildTrajectoryGrader implements Grader { name: "rust-cargo-build-failure-check", description: "Rust-specific grader: checks trajectory for failed cargo build tool calls, extracts Rust compiler errors (E0XXX), and scales score by failure ratio", - behavior: { execution: "single", requiresWorkspace: false }, + behavior: { requiresWorkspace: false }, determinism: "static", - portability: "t3a-scenario", reference: "reference-free", temporalScope: "trajectory-level", costProfile: "free", diff --git a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json index 7caae95e..839e049e 100644 --- a/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json +++ b/tests/scenarios/_shared/vally/grader-plugins/rust-cargo-build-failure/package.json @@ -10,7 +10,7 @@ "typecheck": "tsc --noEmit" }, "devDependencies": { - "@microsoft/vally": "^0.6.0", + "@microsoft/vally": "^0.7.0", "@types/node": "^20.0.0", "typescript": "^5.4.5" }, diff --git a/tests/scenarios/_shared/vally/tools/check-python-idiomatic.mjs b/tests/scenarios/_shared/vally/tools/check-python-idiomatic.mjs new file mode 100644 index 00000000..b2d4bd52 --- /dev/null +++ b/tests/scenarios/_shared/vally/tools/check-python-idiomatic.mjs @@ -0,0 +1,60 @@ +// Heuristic checks for non-idiomatic Python patterns. +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join } from "node:path"; + +const EXCLUDED_DIRS = new Set([ + ".git", + ".hg", + ".svn", + ".venv", + "venv", + "__pycache__", + "node_modules", + ".vally", +]); + +function collectPythonFiles(dir, acc) { + for (const entry of readdirSync(dir)) { + if (entry.startsWith(".") && entry !== ".vally") continue; + if (EXCLUDED_DIRS.has(entry)) continue; + const full = join(dir, entry); + const st = statSync(full); + if (st.isDirectory()) { + collectPythonFiles(full, acc); + } else if (entry.endsWith(".py")) { + acc.push(full); + } + } +} + +const files = []; +collectPythonFiles(process.cwd(), files); + +if (files.length === 0) { + console.error("No Python files found to evaluate for idiomatic checks."); + process.exit(1); +} + +const failures = []; +for (const file of files) { + const text = readFileSync(file, "utf-8"); + if (text.includes("\t")) { + failures.push(`${file}: contains tab indentation`); + } + if (/^\s*from\s+\S+\s+import\s+\*/m.test(text)) { + failures.push(`${file}: uses wildcard import`); + } + if (/^\s*except\s*:\s*$/m.test(text)) { + failures.push(`${file}: uses bare except`); + } +} + +if (failures.length > 0) { + console.error("Non-idiomatic Python patterns found:"); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log(`No non-idiomatic patterns detected in ${files.length} Python file(s).`); diff --git a/tests/scenarios/_shared/vally/tools/check-python-syntax.mjs b/tests/scenarios/_shared/vally/tools/check-python-syntax.mjs new file mode 100644 index 00000000..3542ffff --- /dev/null +++ b/tests/scenarios/_shared/vally/tools/check-python-syntax.mjs @@ -0,0 +1,100 @@ +// Validates syntax for generated Python files. +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import { spawnSync } from "node:child_process"; + +const EXCLUDED_DIRS = new Set([ + ".git", + ".hg", + ".svn", + ".venv", + "venv", + "__pycache__", + "node_modules", + ".vally", +]); + +// Read optional config file for file-level exclusions +let excludePatterns = []; +const configPath = process.argv[2]; +if (configPath) { + try { + const configContent = readFileSync(configPath, "utf-8"); + const config = JSON.parse(configContent); + excludePatterns = config.excludePatterns || []; + } catch (err) { + console.error(`Warning: Could not read config file "${configPath}":`, err.message); + } +} + +function matchGlobPattern(filePath, pattern) { + // Simple glob pattern matcher (supports * and **) + const regexPattern = pattern + .replace(/\./g, "\\.") + .replace(/\*\*/g, "___DOUBLE_STAR___") + .replace(/\*/g, "[^/]*") + .replace(/___DOUBLE_STAR___/g, ".*"); + const regex = new RegExp(`^${regexPattern}$`); + return regex.test(filePath); +} + +function shouldExcludeFile(filePath, baseDir) { + const relativePath = relative(baseDir, filePath).replace(/\\/g, "/"); + for (const pattern of excludePatterns) { + if (matchGlobPattern(relativePath, pattern)) { + return true; + } + } + return false; +} + +function collectPythonFiles(dir, acc, baseDir) { + for (const entry of readdirSync(dir)) { + if (entry.startsWith(".") && entry !== ".vally") continue; + if (EXCLUDED_DIRS.has(entry)) continue; + const full = join(dir, entry); + const st = statSync(full); + if (st.isDirectory()) { + collectPythonFiles(full, acc, baseDir); + } else if (entry.endsWith(".py")) { + if (!shouldExcludeFile(full, baseDir)) { + acc.push(full); + } + } + } +} + +const cwd = process.cwd(); +const files = []; +collectPythonFiles(cwd, files, cwd); + +if (files.length === 0) { + console.error("No Python files found to validate."); + process.exit(1); +} + +const failures = []; +for (const file of files) { + const result = spawnSync("python", ["-m", "py_compile", file], { + encoding: "utf-8", + }); + if (result.status !== 0) { + failures.push({ file, stderr: result.stderr?.trim() ?? "" }); + } +} + +if (failures.length > 0) { + console.error("Python syntax validation failed:"); + for (const failure of failures) { + console.error(`- ${failure.file}`); + if (failure.stderr) { + console.error(failure.stderr); + } + } + process.exit(1); +} + +const message = `Validated syntax for ${files.length} Python file(s).${ + excludePatterns.length > 0 ? ` (${excludePatterns.length} pattern(s) excluded)` : "" +}`; +console.log(message); diff --git a/tests/scenarios/agent-framework-azure-ai-py/scenarios.yaml b/tests/scenarios/agent-framework-azure-ai-py/scenarios.yaml index 055a05b5..12b3e0f1 100644 --- a/tests/scenarios/agent-framework-azure-ai-py/scenarios.yaml +++ b/tests/scenarios/agent-framework-azure-ai-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 @@ -18,7 +18,7 @@ scenarios: - "create_agent" - "run" forbidden_patterns: - - "AgentsClient" # Should use provider for basic example + - "AgentsClient" # Should use provider for basic example tags: - basic - provider @@ -26,7 +26,7 @@ scenarios: import asyncio from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -38,7 +38,7 @@ scenarios: ) result = await agent.run("Hello!") print(result.text) - + asyncio.run(main()) # Persistent agent and thread usage @@ -59,7 +59,7 @@ scenarios: from agent_framework import AgentThread from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -71,7 +71,7 @@ scenarios: print(result.text) print(thread.conversation_id) print(thread.service_thread_id) - + asyncio.run(main()) # Hosted code interpreter tool @@ -84,7 +84,7 @@ scenarios: - "HostedFileContent" - "tools=code_tool" forbidden_patterns: - - "code_interpreter" # Avoid string tool names + - "code_interpreter" # Avoid string tool names tags: - tools - code-interpreter @@ -93,7 +93,7 @@ scenarios: from agent_framework import HostedCodeInterpreterTool, HostedFileContent from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -109,7 +109,7 @@ scenarios: ) result = await agent.run("Summarize the file") print(result.text) - + asyncio.run(main()) # Hosted file search with vector store @@ -133,7 +133,7 @@ scenarios: from agent_framework.azure import AzureAIAgentsProvider from azure.ai.agents.aio import AgentsClient from azure.identity.aio import AzureCliCredential - + async def main(): endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"] async with ( @@ -160,7 +160,7 @@ scenarios: ) result = await agent.run("What does the document say?") print(result.text) - + asyncio.run(main()) # Hosted web search tool @@ -180,7 +180,7 @@ scenarios: from agent_framework import HostedWebSearchTool from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): os.environ["BING_CONNECTION_ID"] = "your-bing-connection-id" async with ( @@ -197,7 +197,7 @@ scenarios: ) result = await agent.run("What are the latest AI updates?") print(result.text) - + asyncio.run(main()) # MCP Streamable HTTP tool @@ -210,7 +210,7 @@ scenarios: - "async with" - "tools=mcp_tool" forbidden_patterns: - - "HostedMCPTool" # Use streamable tool in this scenario + - "HostedMCPTool" # Use streamable tool in this scenario tags: - mcp - tools @@ -219,7 +219,7 @@ scenarios: from agent_framework import MCPStreamableHTTPTool from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -236,7 +236,7 @@ scenarios: ) result = await agent.run("What is Azure AI Foundry?") print(result.text) - + asyncio.run(main()) # Hosted MCP tool @@ -249,7 +249,7 @@ scenarios: - "approval_mode" - "allowed_tools" forbidden_patterns: - - "MCPStreamableHTTPTool" # Keep hosted MCP isolated + - "MCPStreamableHTTPTool" # Keep hosted MCP isolated tags: - mcp - tools @@ -258,7 +258,7 @@ scenarios: from agent_framework import HostedMCPTool from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -276,7 +276,7 @@ scenarios: ) result = await agent.run("How do I use Azure Functions?") print(result.text) - + asyncio.run(main()) # Streaming responses @@ -289,14 +289,14 @@ scenarios: - "async for" - "chunk.text" forbidden_patterns: - - "stream=True" # Use run_stream instead + - "stream=True" # Use run_stream instead tags: - streaming mock_response: | import asyncio from agent_framework.azure import AzureAIAgentsProvider from azure.identity.aio import AzureCliCredential - + async def main(): async with ( AzureCliCredential() as credential, @@ -311,5 +311,5 @@ scenarios: if chunk.text: print(chunk.text, end="", flush=True) print() - + asyncio.run(main()) diff --git a/tests/scenarios/agent-framework-azure-ai-py/test-output.txt b/tests/scenarios/agent-framework-azure-ai-py/test-output.txt new file mode 100644 index 00000000..c07b0b91 --- /dev/null +++ b/tests/scenarios/agent-framework-azure-ai-py/test-output.txt @@ -0,0 +1 @@ +Error: Experiment config file not found: Q:\src\skills\tests\scenarios\agent-framework-azure-ai-py\skill_effectiveness_experiment.yaml diff --git a/tests/scenarios/agent-framework-azure-ai-py/vally/eval.yaml b/tests/scenarios/agent-framework-azure-ai-py/vally/eval.yaml new file mode 100644 index 00000000..9c9e09ff --- /dev/null +++ b/tests/scenarios/agent-framework-azure-ai-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: agent-framework-azure-ai-py-eval +description: Vally evaluation suite for agent-framework-azure-ai-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_provider_agent + prompt: |- + Create a basic Azure AI agent using AzureAIAgentsProvider. + Use async context managers, run a message, and print the response. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap agent initialization and all agent method calls in try/except blocks + * Handle: ServiceError, HttpResponseError, ClientAuthenticationError, generic Exception + * Example: try: run = client.agents.create_run(...) except ServiceError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: persistent_agent_thread + prompt: |- + Show how to retrieve an existing agent and continue a conversation + using a persistent thread. Include conversation ID handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap agent initialization and all agent method calls in try/except blocks + * Handle: ServiceError, HttpResponseError, ClientAuthenticationError, generic Exception + * Example: try: run = client.agents.create_run(...) except ServiceError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - agent-framework-azure-ai-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..fb91afc5 --- /dev/null +++ b/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: agent-framework-azure-ai-py-eval +description: Vally evaluation suite for agent-framework-azure-ai-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_provider_agent + prompt: |- + Create a basic Azure AI agent using AzureAIAgentsProvider. + Use async context managers, run a message, and print the response. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap agent initialization and all agent method calls in try/except blocks + * Handle: ServiceError, HttpResponseError, ClientAuthenticationError, generic Exception + * Example: try: run = client.agents.create_run(...) except ServiceError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: persistent_agent_thread + prompt: |- + Show how to retrieve an existing agent and continue a conversation + using a persistent thread. Include conversation ID handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap agent initialization and all agent method calls in try/except blocks + * Handle: ServiceError, HttpResponseError, ClientAuthenticationError, generic Exception + * Example: try: run = client.agents.create_run(...) except ServiceError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - agent-framework-azure-ai-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..2a2cb90e --- /dev/null +++ b/tests/scenarios/agent-framework-azure-ai-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: agent-framework-azure-ai-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/agent-framework-azure-ai-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-contentsafety-py/scenarios.yaml b/tests/scenarios/azure-ai-contentsafety-py/scenarios.yaml index c8e4515b..9c69fcda 100644 --- a/tests/scenarios/azure-ai-contentsafety-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-contentsafety-py/scenarios.yaml @@ -4,7 +4,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-contentsafety-py/vally/eval.yaml b/tests/scenarios/azure-ai-contentsafety-py/vally/eval.yaml new file mode 100644 index 00000000..5775dc0f --- /dev/null +++ b/tests/scenarios/azure-ai-contentsafety-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-ai-contentsafety-py-eval +description: Vally evaluation suite for azure-ai-contentsafety-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: text_analysis_api_key + prompt: |- + Analyze a piece of text with Azure AI Content Safety using API key + authentication. Include category severity checks for hate, self-harm, + sexual, and violence. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: text_analysis_entra_id + prompt: |- + Analyze text using Microsoft Entra ID authentication with + DefaultAzureCredential. Include analyze_text usage. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-contentsafety-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..6470ab32 --- /dev/null +++ b/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-contentsafety-py-eval +description: Vally evaluation suite for azure-ai-contentsafety-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: text_analysis_api_key + prompt: |- + Analyze a piece of text with Azure AI Content Safety using API key + authentication. Include category severity checks for hate, self-harm, + sexual, and violence. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: text_analysis_entra_id + prompt: |- + Analyze text using Microsoft Entra ID authentication with + DefaultAzureCredential. Include analyze_text usage. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-contentsafety-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..9eb863a1 --- /dev/null +++ b/tests/scenarios/azure-ai-contentsafety-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-contentsafety-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentsafety-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-contentunderstanding-py/scenarios.yaml b/tests/scenarios/azure-ai-contentunderstanding-py/scenarios.yaml index 2d4f9c92..561a65da 100644 --- a/tests/scenarios/azure-ai-contentunderstanding-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-contentunderstanding-py/scenarios.yaml @@ -4,7 +4,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-contentunderstanding-py/vally/eval.yaml b/tests/scenarios/azure-ai-contentunderstanding-py/vally/eval.yaml new file mode 100644 index 00000000..a0db9a59 --- /dev/null +++ b/tests/scenarios/azure-ai-contentunderstanding-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-contentunderstanding-py-eval +description: Vally evaluation suite for azure-ai-contentunderstanding-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_authentication + prompt: |- + Create a basic Content Understanding client using DefaultAzureCredential. + Use the environment variable CONTENTUNDERSTANDING_ENDPOINT. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: gpt-5.5 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: document_extraction_url + prompt: |- + Analyze a PDF document URL using prebuilt-documentSearch and print the markdown. + Use begin_analyze with AnalyzeInput. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: gpt-5.5 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-contentunderstanding-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..2cd04cc1 --- /dev/null +++ b/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-contentunderstanding-py-eval +description: Vally evaluation suite for azure-ai-contentunderstanding-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_authentication + prompt: |- + Create a basic Content Understanding client using DefaultAzureCredential. + Use the environment variable CONTENTUNDERSTANDING_ENDPOINT. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: document_extraction_url + prompt: |- + Analyze a PDF document URL using prebuilt-documentSearch and print the markdown. + Use begin_analyze with AnalyzeInput. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-contentunderstanding-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..a97b7096 --- /dev/null +++ b/tests/scenarios/azure-ai-contentunderstanding-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-contentunderstanding-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-contentunderstanding-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-language-conversations-py/acceptance-criteria.md b/tests/scenarios/azure-ai-language-conversations-py/acceptance-criteria.md index 03fda1f5..31ea369d 100644 --- a/tests/scenarios/azure-ai-language-conversations-py/acceptance-criteria.md +++ b/tests/scenarios/azure-ai-language-conversations-py/acceptance-criteria.md @@ -1,16 +1,69 @@ -# Acceptance Criteria +# Acceptance Criteria: azure-ai-language-conversations-py ## Authentication and Setup -- [ ] MUST use `AzureKeyCredential` from `azure.core.credentials` for authentication. -- [ ] MUST initialize `ConversationAnalysisClient` correctly with an endpoint and credential. -- [ ] MUST NOT hardcode API keys or endpoints in the code examples. + +### ✅ Correct +```python +from azure.identity import DefaultAzureCredential +from azure.ai.language.conversations import ConversationAnalysisClient + +with ConversationAnalysisClient(endpoint, DefaultAzureCredential()) as client: + ... +``` + +### ✅ Correct: legacy key path for existing keyed deployments +```python +from azure.core.credentials import AzureKeyCredential +from azure.ai.language.conversations import ConversationAnalysisClient + +with ConversationAnalysisClient(endpoint, AzureKeyCredential(key)) as client: + ... +``` + +### ❌ Incorrect +```python +client = ConversationAnalysisClient(endpoint, credential) +# Missing context manager +``` ## Payload Construction -- [ ] MUST properly define the `task` dictionary with `"kind": "Conversation"`. -- [ ] MUST include a correctly structured `analysisInput` with `conversationItem` details (id, participantId, modality, text). -- [ ] MUST include required `parameters` (`projectName` and `deploymentName`). - -## API Usage and Extraction -- [ ] MUST demonstrate calling `analyze_conversation`. -- [ ] MUST show how to correctly parse the response dictionary to extract the `topIntent` or entities. -- [ ] MUST use a context manager (`with client:`) for the client instance. \ No newline at end of file + +### ✅ Correct +```python +task = { + "kind": "Conversation", + "analysisInput": { + "conversationItem": { + "participantId": "1", + "id": "1", + "modality": "text", + "text": query, + } + }, + "parameters": { + "projectName": project_name, + "deploymentName": deployment_name, + }, +} +``` + +### ❌ Incorrect +```python +task = { + "kind": "conversations", # Wrong kind + "parameters": {}, +} +``` + +## API Usage + +### ✅ Correct +```python +result = client.analyze_conversation(task=task) +print(result["result"]["prediction"]["topIntent"]) +``` + +### ❌ Incorrect +```python +result = client.analyze(task=task) # Wrong method name +``` diff --git a/tests/scenarios/azure-ai-language-conversations-py/scenarios.yaml b/tests/scenarios/azure-ai-language-conversations-py/scenarios.yaml new file mode 100644 index 00000000..2a396471 --- /dev/null +++ b/tests/scenarios/azure-ai-language-conversations-py/scenarios.yaml @@ -0,0 +1,102 @@ +config: + model: gpt-5.5 + max_tokens: 2000 + temperature: 0.3 + +scenarios: + - name: clu_basic_defaultazurecredential + prompt: | + Create a Python example that uses ConversationAnalysisClient with DefaultAzureCredential. + Build a valid task payload for Conversation analysis and print the top intent. + Use a context manager for the client. + expected_patterns: + - "DefaultAzureCredential" + - "ConversationAnalysisClient" + - "with ConversationAnalysisClient" + - "analyze_conversation" + - '"kind": "Conversation"' + - "projectName" + - "deploymentName" + - "topIntent" + forbidden_patterns: + - "client = ConversationAnalysisClient(" + - "analyze(" + tags: + - basic + - authentication + - payload + mock_response: | + import os + from azure.identity import DefaultAzureCredential + from azure.ai.language.conversations import ConversationAnalysisClient + + endpoint = os.environ["AZURE_CONVERSATIONS_ENDPOINT"] + project_name = os.environ["AZURE_CONVERSATIONS_PROJECT"] + deployment_name = os.environ["AZURE_CONVERSATIONS_DEPLOYMENT"] + + task = { + "kind": "Conversation", + "analysisInput": { + "conversationItem": { + "participantId": "1", + "id": "1", + "modality": "text", + "language": "en", + "text": "Book a flight to Seattle", + } + }, + "parameters": { + "projectName": project_name, + "deploymentName": deployment_name, + "verbose": True, + }, + } + + with ConversationAnalysisClient(endpoint, DefaultAzureCredential()) as client: + result = client.analyze_conversation(task=task) + print(result["result"]["prediction"]["topIntent"]) + + - name: clu_legacy_api_key_existing_deployments + prompt: | + Show the legacy API-key authentication path for an existing keyed deployment. + Use AzureKeyCredential with ConversationAnalysisClient in a context manager. + Mention this path is for existing deployments. + expected_patterns: + - "AzureKeyCredential" + - "ConversationAnalysisClient" + - "with ConversationAnalysisClient" + - "analyze_conversation" + forbidden_patterns: + - 'api_key="' + - "client = ConversationAnalysisClient(" + tags: + - legacy + - authentication + mock_response: | + import os + from azure.core.credentials import AzureKeyCredential + from azure.ai.language.conversations import ConversationAnalysisClient + + endpoint = os.environ["AZURE_CONVERSATIONS_ENDPOINT"] + key = os.environ["AZURE_CONVERSATIONS_KEY"] + + # Legacy path for existing keyed deployments; prefer DefaultAzureCredential for new code. + with ConversationAnalysisClient(endpoint, AzureKeyCredential(key)) as client: + result = client.analyze_conversation( + task={ + "kind": "Conversation", + "analysisInput": { + "conversationItem": { + "participantId": "1", + "id": "1", + "modality": "text", + "text": "What is my order status?", + } + }, + "parameters": { + "projectName": os.environ["AZURE_CONVERSATIONS_PROJECT"], + "deploymentName": os.environ["AZURE_CONVERSATIONS_DEPLOYMENT"], + }, + } + ) + print(result["result"]["prediction"]["topIntent"]) diff --git a/tests/scenarios/azure-ai-language-conversations-py/vally/eval.yaml b/tests/scenarios/azure-ai-language-conversations-py/vally/eval.yaml new file mode 100644 index 00000000..e143ffdf --- /dev/null +++ b/tests/scenarios/azure-ai-language-conversations-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-language-conversations-py-eval +description: Vally evaluation suite for azure-ai-language-conversations-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: clu_basic_defaultazurecredential + prompt: |- + Create a Python example that uses ConversationAnalysisClient with DefaultAzureCredential. + Build a valid task payload for Conversation analysis and print the top intent. + Use a context manager for the client. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: clu_legacy_api_key_existing_deployments + prompt: |- + Show the legacy API-key authentication path for an existing keyed deployment. + Use AzureKeyCredential with ConversationAnalysisClient in a context manager. + Mention this path is for existing deployments. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-language-conversations-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..39bd90f3 --- /dev/null +++ b/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-language-conversations-py-eval +description: Vally evaluation suite for azure-ai-language-conversations-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: clu_basic_defaultazurecredential + prompt: |- + Create a Python example that uses ConversationAnalysisClient with DefaultAzureCredential. + Build a valid task payload for Conversation analysis and print the top intent. + Use a context manager for the client. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: clu_legacy_api_key_existing_deployments + prompt: |- + Show the legacy API-key authentication path for an existing keyed deployment. + Use AzureKeyCredential with ConversationAnalysisClient in a context manager. + Mention this path is for existing deployments. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-language-conversations-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..de554732 --- /dev/null +++ b/tests/scenarios/azure-ai-language-conversations-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-language-conversations-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-language-conversations-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-ml-py/scenarios.yaml b/tests/scenarios/azure-ai-ml-py/scenarios.yaml index 9daa0b24..4ec9baa5 100644 --- a/tests/scenarios/azure-ai-ml-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-ml-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-ml-py/vally/eval.yaml b/tests/scenarios/azure-ai-ml-py/vally/eval.yaml new file mode 100644 index 00000000..60c158b3 --- /dev/null +++ b/tests/scenarios/azure-ai-ml-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-ai-ml-py-eval +description: Vally evaluation suite for azure-ai-ml-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: mlclient_creation_with_env_vars + prompt: |- + Create an MLClient instance using environment variables for authentication. + Include proper DefaultAzureCredential usage, and context manager pattern + for resource cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: create_and_list_workspaces + prompt: |- + Create a new Azure ML workspace with proper configuration, then list + all workspaces in the resource group. Include proper error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-ml-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-ml-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..235fa737 --- /dev/null +++ b/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-ml-py-eval +description: Vally evaluation suite for azure-ai-ml-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: mlclient_creation_with_env_vars + prompt: |- + Create an MLClient instance using environment variables for authentication. + Include proper DefaultAzureCredential usage, and context manager pattern + for resource cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: create_and_list_workspaces + prompt: |- + Create a new Azure ML workspace with proper configuration, then list + all workspaces in the resource group. Include proper error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-ml-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-ml-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_experiment.yaml similarity index 76% rename from tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_experiment.yaml rename to tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_experiment.yaml index bb5e04c9..de02a843 100644 --- a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_experiment.yaml +++ b/tests/scenarios/azure-ai-ml-py/vally/skill_effectiveness_experiment.yaml @@ -1,21 +1,18 @@ # Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. -name: azure-servicebus-rust-skill-experiment +name: azure-ai-ml-py-skill-experiment evals: - skill_effectiveness_eval.yaml - vary: - /defaults/model - /environment/skills - baseline: sonnet_baseline - variants: sonnet_skill: overrides: model: claude-sonnet-4.6 environment: skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-ml-py sonnet_baseline: overrides: model: claude-sonnet-4.6 diff --git a/tests/scenarios/azure-ai-projects-py/scenarios.yaml b/tests/scenarios/azure-ai-projects-py/scenarios.yaml index fd92a540..28f7d377 100644 --- a/tests/scenarios/azure-ai-projects-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-projects-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-projects-py/vally/eval.yaml b/tests/scenarios/azure-ai-projects-py/vally/eval.yaml new file mode 100644 index 00000000..d3975725 --- /dev/null +++ b/tests/scenarios/azure-ai-projects-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-ai-projects-py-eval +description: Vally evaluation suite for azure-ai-projects-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: client_creation + prompt: |- + Create an AIProjectClient with proper authentication using DefaultAzureCredential. + Use environment variables for the endpoint and include a context manager pattern. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: versioned_agent_creation + prompt: |- + Create a versioned agent using create_version() with PromptAgentDefinition. + Include a version label and description. The agent should be named "customer-support-agent". + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-projects-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-projects-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..752ced17 --- /dev/null +++ b/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-projects-py-eval +description: Vally evaluation suite for azure-ai-projects-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: client_creation + prompt: |- + Create an AIProjectClient with proper authentication using DefaultAzureCredential. + Use environment variables for the endpoint and include a context manager pattern. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: versioned_agent_creation + prompt: |- + Create a versioned agent using create_version() with PromptAgentDefinition. + Include a version label and description. The agent should be named "customer-support-agent". + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-projects-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-projects-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..a758a0af --- /dev/null +++ b/tests/scenarios/azure-ai-projects-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-projects-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-projects-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-textanalytics-py/scenarios.yaml b/tests/scenarios/azure-ai-textanalytics-py/scenarios.yaml index 089b9869..0b8f1fdc 100644 --- a/tests/scenarios/azure-ai-textanalytics-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-textanalytics-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-textanalytics-py/vally/eval.yaml b/tests/scenarios/azure-ai-textanalytics-py/vally/eval.yaml new file mode 100644 index 00000000..ffe80c79 --- /dev/null +++ b/tests/scenarios/azure-ai-textanalytics-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-textanalytics-py-eval +description: Vally evaluation suite for azure-ai-textanalytics-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_sentiment_analysis + prompt: |- + Create a basic Azure Text Analytics example that analyzes sentiment + of multiple documents. Include proper authentication with DefaultAzureCredential, + error handling, and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: sentiment_with_opinion_mining + prompt: |- + Create an example that analyzes sentiment with opinion mining (aspect-based sentiment). + Extract both the target and assessments for each opinion. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-textanalytics-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + + diff --git a/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..8371375c --- /dev/null +++ b/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-textanalytics-py-eval +description: Vally evaluation suite for azure-ai-textanalytics-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_sentiment_analysis + prompt: |- + Create a basic Azure Text Analytics example that analyzes sentiment + of multiple documents. Include proper authentication with DefaultAzureCredential, + error handling, and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: sentiment_with_opinion_mining + prompt: |- + Create an example that analyzes sentiment with opinion mining (aspect-based sentiment). + Extract both the target and assessments for each opinion. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-textanalytics-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..e1791f10 --- /dev/null +++ b/tests/scenarios/azure-ai-textanalytics-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-textanalytics-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-textanalytics-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-transcription-py/scenarios.yaml b/tests/scenarios/azure-ai-transcription-py/scenarios.yaml index b7c3005b..b6fb1878 100644 --- a/tests/scenarios/azure-ai-transcription-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-transcription-py/scenarios.yaml @@ -1,6 +1,6 @@ config: max_tokens: 2000 - model: gpt-4 + model: gpt-5.5 temperature: 0.3 scenarios: - expected_patterns: diff --git a/tests/scenarios/azure-ai-transcription-py/vally/eval.yaml b/tests/scenarios/azure-ai-transcription-py/vally/eval.yaml new file mode 100644 index 00000000..8754576a --- /dev/null +++ b/tests/scenarios/azure-ai-transcription-py/vally/eval.yaml @@ -0,0 +1,86 @@ +name: azure-ai-transcription-py-eval +description: Vally evaluation suite for azure-ai-transcription-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-ai-transcription-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-transcription-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..0a7b3f12 --- /dev/null +++ b/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,87 @@ +name: azure-ai-transcription-py-eval +description: Vally evaluation suite for azure-ai-transcription-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-ai-transcription-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-transcription-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..22878bab --- /dev/null +++ b/tests/scenarios/azure-ai-transcription-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-transcription-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-transcription-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-translation-document-py/scenarios.yaml b/tests/scenarios/azure-ai-translation-document-py/scenarios.yaml index f4f5a761..4a739c77 100644 --- a/tests/scenarios/azure-ai-translation-document-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-translation-document-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-translation-document-py/vally/eval.yaml b/tests/scenarios/azure-ai-translation-document-py/vally/eval.yaml new file mode 100644 index 00000000..5b324a05 --- /dev/null +++ b/tests/scenarios/azure-ai-translation-document-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-ai-translation-document-py-eval +description: Vally evaluation suite for azure-ai-translation-document-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_batch_translation + prompt: |- + Create a script that submits a batch of documents for translation from a + source Azure Storage container to a target container. Show proper client + initialization and job submission. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: multiple_target_languages + prompt: |- + Create a script that translates a batch of documents to multiple languages + (Spanish, French, German) in a single translation job. Show how to specify + different target containers for each language. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-translation-document-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..fe92089b --- /dev/null +++ b/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-translation-document-py-eval +description: Vally evaluation suite for azure-ai-translation-document-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_batch_translation + prompt: |- + Create a script that submits a batch of documents for translation from a + source Azure Storage container to a target container. Show proper client + initialization and job submission. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: multiple_target_languages + prompt: |- + Create a script that translates a batch of documents to multiple languages + (Spanish, French, German) in a single translation job. Show how to specify + different target containers for each language. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-translation-document-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..0d5dc06d --- /dev/null +++ b/tests/scenarios/azure-ai-translation-document-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-translation-document-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-document-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-translation-text-py/scenarios.yaml b/tests/scenarios/azure-ai-translation-text-py/scenarios.yaml index 9e4b96cb..ce31fcaa 100644 --- a/tests/scenarios/azure-ai-translation-text-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-translation-text-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-translation-text-py/vally/eval.yaml b/tests/scenarios/azure-ai-translation-text-py/vally/eval.yaml new file mode 100644 index 00000000..2b196804 --- /dev/null +++ b/tests/scenarios/azure-ai-translation-text-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-translation-text-py-eval +description: Vally evaluation suite for azure-ai-translation-text-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_text_translation + prompt: |- + Create a basic example that translates English text to Spanish and French. + Include proper authentication with Entra ID credentials, client initialization, + and proper handling of the translation results. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: language_detection + prompt: |- + Create an example that detects the language of provided text using the + Azure AI Text Translation service. Show both automatic detection via + translate and explicit detection via the detect method. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-translation-text-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d9febe94 --- /dev/null +++ b/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-translation-text-py-eval +description: Vally evaluation suite for azure-ai-translation-text-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_text_translation + prompt: |- + Create a basic example that translates English text to Spanish and French. + Include proper authentication with Entra ID credentials, client initialization, + and proper handling of the translation results. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: language_detection + prompt: |- + Create an example that detects the language of provided text using the + Azure AI Text Translation service. Show both automatic detection via + translate and explicit detection via the detect method. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-translation-text-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..d263c7f5 --- /dev/null +++ b/tests/scenarios/azure-ai-translation-text-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-translation-text-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-translation-text-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-vision-imageanalysis-py/scenarios.yaml b/tests/scenarios/azure-ai-vision-imageanalysis-py/scenarios.yaml index 63f2290f..fa4fc082 100644 --- a/tests/scenarios/azure-ai-vision-imageanalysis-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-vision-imageanalysis-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/eval.yaml b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/eval.yaml new file mode 100644 index 00000000..a82d6eb1 --- /dev/null +++ b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-vision-imageanalysis-py-eval +description: Vally evaluation suite for azure-ai-vision-imageanalysis-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: image_captioning_from_url + prompt: |- + Create a basic example that analyzes an image from a URL to generate a caption. + Include proper authentication with DefaultAzureCredential, context manager for cleanup, + and extract the caption text with confidence score. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: ocr_text_extraction + prompt: |- + Create an example that extracts text from an image using OCR. + Include handling the READ visual feature to get text blocks, lines, and words. + Display text with confidence scores and bounding polygons. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-vision-imageanalysis-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..b19ecaa0 --- /dev/null +++ b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-ai-vision-imageanalysis-py-eval +description: Vally evaluation suite for azure-ai-vision-imageanalysis-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: image_captioning_from_url + prompt: |- + Create a basic example that analyzes an image from a URL to generate a caption. + Include proper authentication with DefaultAzureCredential, context manager for cleanup, + and extract the caption text with confidence score. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: ocr_text_extraction + prompt: |- + Create an example that extracts text from an image using OCR. + Include handling the READ visual feature to get text blocks, lines, and words. + Display text with confidence scores and bounding polygons. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-vision-imageanalysis-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..e9ecc2b1 --- /dev/null +++ b/tests/scenarios/azure-ai-vision-imageanalysis-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-vision-imageanalysis-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-vision-imageanalysis-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-ai-voicelive-py/scenarios.yaml b/tests/scenarios/azure-ai-voicelive-py/scenarios.yaml index a7a623b4..fb1d5eab 100644 --- a/tests/scenarios/azure-ai-voicelive-py/scenarios.yaml +++ b/tests/scenarios/azure-ai-voicelive-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-ai-voicelive-py/vally/eval.yaml b/tests/scenarios/azure-ai-voicelive-py/vally/eval.yaml new file mode 100644 index 00000000..f967ec45 --- /dev/null +++ b/tests/scenarios/azure-ai-voicelive-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-ai-voicelive-py-eval +description: Vally evaluation suite for azure-ai-voicelive-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_session_setup + prompt: |- + Create a minimal async Voice Live connection that configures a session + using RequestSession with instructions, modalities, and Server VAD. + Use DefaultAzureCredential and update the session. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: api_key_authentication + prompt: |- + Connect to Voice Live using AzureKeyCredential and configure a session + with text+audio modalities. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-voicelive-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..c5befe75 --- /dev/null +++ b/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-ai-voicelive-py-eval +description: Vally evaluation suite for azure-ai-voicelive-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_session_setup + prompt: |- + Create a minimal async Voice Live connection that configures a session + using RequestSession with instructions, modalities, and Server VAD. + Use DefaultAzureCredential and update the session. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: api_key_authentication + prompt: |- + Connect to Voice Live using AzureKeyCredential and configure a session + with text+audio modalities. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-ai-voicelive-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..1fafb4dd --- /dev/null +++ b/tests/scenarios/azure-ai-voicelive-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-ai-voicelive-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-ai-voicelive-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-appconfiguration-py/scenarios.yaml b/tests/scenarios/azure-appconfiguration-py/scenarios.yaml index bfdd9e22..869ff46d 100644 --- a/tests/scenarios/azure-appconfiguration-py/scenarios.yaml +++ b/tests/scenarios/azure-appconfiguration-py/scenarios.yaml @@ -1,5 +1,5 @@ config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 scenarios: diff --git a/tests/scenarios/azure-appconfiguration-py/vally/eval.yaml b/tests/scenarios/azure-appconfiguration-py/vally/eval.yaml new file mode 100644 index 00000000..af458061 --- /dev/null +++ b/tests/scenarios/azure-appconfiguration-py/vally/eval.yaml @@ -0,0 +1,86 @@ +name: azure-appconfiguration-py-eval +description: Vally evaluation suite for azure-appconfiguration-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-appconfiguration-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-appconfiguration-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..92e6387f --- /dev/null +++ b/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,87 @@ +name: azure-appconfiguration-py-eval +description: Vally evaluation suite for azure-appconfiguration-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-appconfiguration-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-appconfiguration-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..993429ab --- /dev/null +++ b/tests/scenarios/azure-appconfiguration-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-appconfiguration-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-appconfiguration-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-containerregistry-py/scenarios.yaml b/tests/scenarios/azure-containerregistry-py/scenarios.yaml index bab3a762..b27b0010 100644 --- a/tests/scenarios/azure-containerregistry-py/scenarios.yaml +++ b/tests/scenarios/azure-containerregistry-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-containerregistry-py/vally/eval.yaml b/tests/scenarios/azure-containerregistry-py/vally/eval.yaml new file mode 100644 index 00000000..484f5737 --- /dev/null +++ b/tests/scenarios/azure-containerregistry-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-containerregistry-py-eval +description: Vally evaluation suite for azure-containerregistry-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_repository_listing + prompt: |- + Create a basic example that lists all repositories in an Azure Container Registry + using DefaultAzureCredential and a context manager for resource cleanup. + Include the registry endpoint from environment variables. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: repository_properties_management + prompt: |- + Write code to get repository properties, update the writeEnabled property to lock + a production repository from being modified, and then display the updated properties. + Use context manager for resource cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-containerregistry-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-containerregistry-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..e104966f --- /dev/null +++ b/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-containerregistry-py-eval +description: Vally evaluation suite for azure-containerregistry-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_repository_listing + prompt: |- + Create a basic example that lists all repositories in an Azure Container Registry + using DefaultAzureCredential and a context manager for resource cleanup. + Include the registry endpoint from environment variables. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: repository_properties_management + prompt: |- + Write code to get repository properties, update the writeEnabled property to lock + a production repository from being modified, and then display the updated properties. + Use context manager for resource cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-containerregistry-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-containerregistry-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..d33a5118 --- /dev/null +++ b/tests/scenarios/azure-containerregistry-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-containerregistry-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-containerregistry-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-cosmos-db-py/scenarios.yaml b/tests/scenarios/azure-cosmos-db-py/scenarios.yaml index c7077927..57b1f846 100644 --- a/tests/scenarios/azure-cosmos-db-py/scenarios.yaml +++ b/tests/scenarios/azure-cosmos-db-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 @@ -28,10 +28,10 @@ scenarios: import os from azure.cosmos import CosmosClient from azure.identity import DefaultAzureCredential - + def _is_emulator_endpoint(endpoint: str) -> bool: return "localhost" in endpoint.lower() or "127.0.0.1" in endpoint - + def create_client(endpoint: str, cosmos_key: str) -> CosmosClient: if _is_emulator_endpoint(endpoint): return CosmosClient( @@ -43,7 +43,7 @@ scenarios: url=endpoint, credential=DefaultAzureCredential(), ) - + client = create_client( endpoint=os.environ["COSMOS_ENDPOINT"], cosmos_key=os.environ.get("COSMOS_KEY", ""), @@ -59,30 +59,98 @@ scenarios: - "_init_attempted" - "get_container" - "container.read()" - forbidden_patterns: - - "CosmosClient(" # client should be created in helper, not per call + - "_create_client" + # forbidden_patterns: + # - "CosmosClient(" # client should be created in helper, not per call tags: - client - singleton mock_response: | + import logging + from threading import Lock from typing import Optional - from azure.cosmos import ContainerProxy - from app.db.cosmos import create_client - from app.core.config import settings - + + from azure.cosmos import ContainerProxy, CosmosClient + from azure.identity import DefaultAzureCredential + + from app.config import settings + + logger = logging.getLogger(__name__) + + _cosmos_client: Optional[CosmosClient] = None _cosmos_container: Optional[ContainerProxy] = None - _init_attempted: bool = False - + _init_attempted = False + _init_lock = Lock() + + + def _is_emulator_endpoint(endpoint: str) -> bool: + return \"localhost\" in endpoint.lower() or \"127.0.0.1\" in endpoint + + + def _create_client() -> CosmosClient: + if _is_emulator_endpoint(settings.cosmos_endpoint): + return CosmosClient( + url=settings.cosmos_endpoint, + credential=settings.cosmos_key, + connection_verify=False, + ) + + return CosmosClient( + url=settings.cosmos_endpoint, + credential=DefaultAzureCredential(), + ) + + def get_container() -> Optional[ContainerProxy]: - global _cosmos_container, _init_attempted + global _cosmos_client, _cosmos_container, _init_attempted + if _init_attempted: return _cosmos_container - _init_attempted = True - client = create_client(settings.cosmos_endpoint, settings.cosmos_key) - database = client.get_database_client(settings.cosmos_database_name) - _cosmos_container = database.get_container_client(settings.cosmos_container_id) - _cosmos_container.read() - return _cosmos_container + + with _init_lock: + if _init_attempted: + return _cosmos_container + + _init_attempted = True + + try: + _cosmos_client = _create_client() + database = _cosmos_client.get_database_client(settings.cosmos_database_name) + _cosmos_container = database.get_container_client(settings.cosmos_container_id) + _cosmos_container.read() + + logger.info( + \"Cosmos DB connected\", + extra={ + \"database\": settings.cosmos_database_name, + \"container\": settings.cosmos_container_id, + \"endpoint\": settings.cosmos_endpoint[:30] + \"...\", + }, + ) + except Exception as exc: + logger.error( + \"Cosmos DB connection failed\", + extra={ + \"error_type\": type(exc).__name__, + \"error_message\": str(exc), + \"endpoint\": settings.cosmos_endpoint[:30] + \"...\", + }, + exc_info=True, + ) + _cosmos_container = None + + return _cosmos_container + + + def reset_connection() -> None: + global _cosmos_client, _cosmos_container, _init_attempted + + if _cosmos_client is not None: + _cosmos_client.close() + + _cosmos_client = None + _cosmos_container = None + _init_attempted = False # Service Layer CRUD - name: service_layer_crud @@ -104,11 +172,11 @@ scenarios: from typing import Optional from app.db.cosmos import get_container, get_document, upsert_document, delete_document, query_documents from app.models.project import Project, ProjectCreate, ProjectUpdate, ProjectInDB - + class ProjectService: def _use_cosmos(self) -> bool: return get_container() is not None - + def _doc_to_model_in_db(self, doc: dict) -> ProjectInDB: return ProjectInDB( id=doc["id"], @@ -116,7 +184,7 @@ scenarios: workspace_id=doc["workspaceId"], doc_type=doc.get("docType", "project"), ) - + def _model_in_db_to_doc(self, model: ProjectInDB) -> dict: return { "id": model.id, @@ -124,10 +192,10 @@ scenarios: "workspaceId": model.workspace_id, "docType": model.doc_type, } - + def _model_in_db_to_model(self, model: ProjectInDB) -> Project: return Project(id=model.id, name=model.name, workspace_id=model.workspace_id) - + async def create(self, data: ProjectCreate, author_id: str) -> Project: if not self._use_cosmos(): raise RuntimeError("Database unavailable") @@ -139,7 +207,7 @@ scenarios: ) await upsert_document(self._model_in_db_to_doc(model), partition_key=data.workspace_id) return self._model_in_db_to_model(model) - + async def get_by_id(self, project_id: str, workspace_id: str) -> Optional[Project]: if not self._use_cosmos(): return None @@ -147,7 +215,7 @@ scenarios: if doc is None: return None return self._model_in_db_to_model(self._doc_to_model_in_db(doc)) - + async def update(self, project_id: str, workspace_id: str, data: ProjectUpdate) -> Optional[Project]: if not self._use_cosmos(): return None @@ -161,18 +229,18 @@ scenarios: setattr(model, field, value) await upsert_document(self._model_in_db_to_doc(model), partition_key=workspace_id) return self._model_in_db_to_model(model) - + async def delete(self, project_id: str, workspace_id: str) -> bool: if not self._use_cosmos(): return False return await delete_document(project_id, partition_key=workspace_id) - + async def list_by_workspace(self, workspace_id: str) -> list[Project]: if not self._use_cosmos(): return [] docs = await query_documents(doc_type="project", partition_key=workspace_id) return [self._model_in_db_to_model(self._doc_to_model_in_db(doc)) for doc in docs] - + project_service = ProjectService() # Parameterized Query @@ -185,14 +253,14 @@ scenarios: - "@slug" - 'parameters=[{"name": "@slug"' forbidden_patterns: - - "f\"SELECT" - - "%s" # string formatting in query + - 'f"SELECT' + - "%s" # string formatting in query tags: - queries - security mock_response: | from app.db.cosmos import query_documents - + async def get_by_slug(slug: str, workspace_id: str): docs = await query_documents( doc_type="project", @@ -212,15 +280,15 @@ scenarios: - "partition_key=workspace_id" - "GLOBAL_PARTITION" forbidden_patterns: - - "partition_key=None" # should use a known partition when possible + - "partition_key=None" # should use a known partition when possible tags: - partitioning mock_response: | GLOBAL_PARTITION = "global" - + async def get_project(project_id: str, workspace_id: str): return await get_document(project_id, partition_key=workspace_id) - + async def get_user_by_email(email: str): return await query_documents( doc_type="user", @@ -245,7 +313,7 @@ scenarios: mock_response: | from starlette.concurrency import run_in_threadpool from app.db.cosmos import get_container - + async def list_all_users(): container = get_container() query = "SELECT * FROM c WHERE c.docType = @docType" @@ -268,7 +336,7 @@ scenarios: - "mocker.patch" - "mock_cosmos_container" forbidden_patterns: - - "CosmosClient(" # unit tests should not initialize real clients + - "CosmosClient(" # unit tests should not initialize real clients tags: - tests - tdd @@ -276,13 +344,13 @@ scenarios: import pytest from unittest.mock import MagicMock from app.services.project_service import project_service - + @pytest.fixture def mock_cosmos_container(mocker): container = MagicMock() mocker.patch("app.db.cosmos.get_container", return_value=container) return container - + @pytest.mark.asyncio async def test_get_by_id_returns_project(mock_cosmos_container): mock_cosmos_container.read_item.return_value = {"id": "123", "name": "Test", "workspaceId": "ws-1"} @@ -307,9 +375,9 @@ scenarios: mock_response: | from fastapi import APIRouter from app.services.project_service import project_service - + router = APIRouter() - + @router.get("/workspaces/{workspace_id}/projects/{project_id}") async def get_project(project_id: str, workspace_id: str): return await project_service.get_by_id(project_id, workspace_id) diff --git a/tests/scenarios/azure-cosmos-db-py/vally/eval.yaml b/tests/scenarios/azure-cosmos-db-py/vally/eval.yaml new file mode 100644 index 00000000..794370b9 --- /dev/null +++ b/tests/scenarios/azure-cosmos-db-py/vally/eval.yaml @@ -0,0 +1,140 @@ +name: azure-cosmos-db-py-eval +description: Vally evaluation suite for azure-cosmos-db-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: dual_auth_client_setup + prompt: |- + Create a Cosmos DB client setup with dual authentication. + Use DefaultAzureCredential in Azure and key-based auth for emulator. + Include emulator detection and SSL verification disabled for emulator. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all CRUD operations in try/except blocks + * Handle: ClientAuthenticationError, CosmosHttpResponseError, AzureError + * Example: try: item = container.upsert_item(body) except CosmosHttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Includes explicit try/except exception handling. + - Provides complete executable Python code with clear control flow. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when: + 1. The code includes try/except blocks (explicit exception handling required) + 2. Uses Pythonic naming and clear control flow + 3. Avoids wildcard imports + 4. Uses context managers or lifecycle-safe patterns where relevant + 5. Uses proper error handling (no bare except clauses) + Fail immediately if no Python code is generated or if there are no try/except blocks. + scoring: scale_1_5 + - name: singleton_container_initialization + prompt: |- + Show a singleton Cosmos container initialization with a cached container + and a safe one-time initialization flag. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all CRUD operations in try/except blocks + * Handle: ClientAuthenticationError, CosmosHttpResponseError, AzureError + * Example: try: item = container.upsert_item(body) except CosmosHttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Includes explicit try/except exception handling. + - Provides complete executable Python code with clear control flow. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when: + 1. The code includes try/except blocks (explicit exception handling required) + 2. Uses Pythonic naming and clear control flow + 3. Avoids wildcard imports + 4. Uses context managers or lifecycle-safe patterns where relevant + 5. Uses proper error handling (no bare except clauses) + Fail immediately if no Python code is generated or if there are no try/except blocks. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-cosmos-db-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..0804934e --- /dev/null +++ b/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,141 @@ +name: azure-cosmos-db-py-eval +description: Vally evaluation suite for azure-cosmos-db-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: dual_auth_client_setup + prompt: |- + Create a Cosmos DB client setup with dual authentication. + Use DefaultAzureCredential in Azure and key-based auth for emulator. + Include emulator detection and SSL verification disabled for emulator. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all CRUD operations in try/except blocks + * Handle: ClientAuthenticationError, CosmosHttpResponseError, AzureError + * Example: try: item = container.upsert_item(body) except CosmosHttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Includes explicit try/except exception handling. + - Provides complete executable Python code with clear control flow. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when: + 1. The code includes try/except blocks (explicit exception handling required) + 2. Uses Pythonic naming and clear control flow + 3. Avoids wildcard imports + 4. Uses context managers or lifecycle-safe patterns where relevant + 5. Uses proper error handling (no bare except clauses) + Fail immediately if no Python code is generated or if there are no try/except blocks. + scoring: scale_1_5 + - name: singleton_container_initialization + prompt: |- + Show a singleton Cosmos container initialization with a cached container + and a safe one-time initialization flag. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all CRUD operations in try/except blocks + * Handle: ClientAuthenticationError, CosmosHttpResponseError, AzureError + * Example: try: item = container.upsert_item(body) except CosmosHttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Includes explicit try/except exception handling. + - Provides complete executable Python code with clear control flow. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when: + 1. The code includes try/except blocks (explicit exception handling required) + 2. Uses Pythonic naming and clear control flow + 3. Avoids wildcard imports + 4. Uses context managers or lifecycle-safe patterns where relevant + 5. Uses proper error handling (no bare except clauses) + Fail immediately if no Python code is generated or if there are no try/except blocks. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-cosmos-db-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..43719fc0 --- /dev/null +++ b/tests/scenarios/azure-cosmos-db-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-cosmos-db-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-db-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-cosmos-py/scenarios.yaml b/tests/scenarios/azure-cosmos-py/scenarios.yaml index dd6000dc..8c64fa1b 100644 --- a/tests/scenarios/azure-cosmos-py/scenarios.yaml +++ b/tests/scenarios/azure-cosmos-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-cosmos-py/vally/eval.yaml b/tests/scenarios/azure-cosmos-py/vally/eval.yaml new file mode 100644 index 00000000..2b5df3b9 --- /dev/null +++ b/tests/scenarios/azure-cosmos-py/vally/eval.yaml @@ -0,0 +1,148 @@ +name: azure-cosmos-py-eval +description: Vally evaluation suite for azure-cosmos-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_aad_client + prompt: |- + Create a basic Cosmos DB NoSQL example using DefaultAzureCredential. + Get a database and container, upsert an item, and read it back. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Ensure ALL Python syntax is valid and correct: + * Use proper bracket matching: {}, [], () + * Do not use placeholder variables or template syntax + - All .py files must pass Python syntax validation: `python -m py_compile ` + - CRITICAL: Include explicit exception handling: + * Wrap credential acquisition in try/except + * Wrap client initialization in try/except + * Wrap all CRUD operations (upsert, read) in try/except blocks + * Handle specific exceptions: ClientAuthenticationError, CosmosHttpResponseError, exceptions.AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: handle_auth_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: connection_string_client + prompt: |- + Create a CosmosClient using a connection string and retrieve + an existing database. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Ensure ALL Python syntax is valid and correct: + * Use proper bracket matching: {}, [], () + * Do not use placeholder variables or template syntax + - All .py files must pass Python syntax validation: `python -m py_compile ` + - CRITICAL: Include explicit exception handling: + * Wrap client initialization in try/except + * Wrap database retrieval in try/except blocks + * Handle specific exceptions: ClientAuthenticationError, CosmosHttpResponseError, exceptions.AzureError + * Example: try: client = CosmosClient.from_connection_string(conn) except ClientAuthenticationError as e: handle_auth_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-cosmos-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..98b0c3c9 --- /dev/null +++ b/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,149 @@ +name: azure-cosmos-py-eval +description: Vally evaluation suite for azure-cosmos-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_aad_client + prompt: |- + Create a basic Cosmos DB NoSQL example using DefaultAzureCredential. + Get a database and container, upsert an item, and read it back. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Ensure ALL Python syntax is valid and correct: + * Use proper bracket matching: {}, [], () + * Do not use placeholder variables or template syntax + - All .py files must pass Python syntax validation: `python -m py_compile ` + - CRITICAL: Include explicit exception handling: + * Wrap credential acquisition in try/except + * Wrap client initialization in try/except + * Wrap all CRUD operations (upsert, read) in try/except blocks + * Handle specific exceptions: ClientAuthenticationError, CosmosHttpResponseError, exceptions.AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: handle_auth_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: connection_string_client + prompt: |- + Create a CosmosClient using a connection string and retrieve + an existing database. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Ensure ALL Python syntax is valid and correct: + * Use proper bracket matching: {}, [], () + * Do not use placeholder variables or template syntax + - All .py files must pass Python syntax validation: `python -m py_compile ` + - CRITICAL: Include explicit exception handling: + * Wrap client initialization in try/except + * Wrap database retrieval in try/except blocks + * Handle specific exceptions: ClientAuthenticationError, CosmosHttpResponseError, exceptions.AzureError + * Example: try: client = CosmosClient.from_connection_string(conn) except ClientAuthenticationError as e: handle_auth_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-cosmos-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..a88954b6 --- /dev/null +++ b/tests/scenarios/azure-cosmos-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-cosmos-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-cosmos-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml deleted file mode 100644 index d0ee97e9..00000000 --- a/tests/scenarios/azure-cosmos-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,181 +0,0 @@ -name: azure-cosmos-rust-eval -description: Evaluation suite for azure-data-cosmos-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-cosmos-crud-operations - prompt: | - Write a Rust program named rust-cosmos-ops that demonstrates complete CRUD operations - and NoSQL document patterns with Azure Cosmos DB: - - Setup operations: - 1. Create a CosmosClient using builder pattern with ProximityTo routing strategy - 2. Get a database client for "myDatabase" - 3. Get a container client for "myContainer" - - CRUD operations: - 1. Create an item with partition key and document fields using create_item() - 2. Read an item by partition key and ID using read_item() - 3. Update an item by replacing it with replace_item() - 4. Delete an item using delete_item() - 5. Handle partition key requirements in all operations - - Advanced operations: - 1. Patch an item using patch_item() with PatchInstructions - 2. Use PatchOperation::set() to update specific fields - 3. Extract model from responses using into_model() - - Data structures: - 1. Define a struct Item with id, partition_key, and value fields - 2. Use serde::Serialize and serde::Deserialize traits - 3. Demonstrate serde_json for JSON operations - - Demonstrate: - - CosmosClient::builder().build(account, routing_strategy).await pattern - - Container hierarchy: client -> database -> container - - Partition key as required parameter in all item operations - - Result-based error handling with ErrorResponse extraction - - TokenCredential authentication (DeveloperToolsCredential/ManagedIdentityCredential) - - Async/await with Tokio runtime - - into_model() for response conversion - - The program should compile and execute successfully, demonstrating the complete - Cosmos DB NoSQL document lifecycle with proper partition key management. - - rubric: - - "Uses official azure_data_cosmos SDK for NoSQL operations." - - "Creates CosmosClient with builder pattern and ProximityTo routing strategy." - - "Navigates client hierarchy: client -> database_client() -> container_client()." - - "Create items with partition key and ID using create_item()." - - "Read items by partition key and ID using read_item()." - - "Update items with replace_item() for full updates." - - "Delete items using delete_item() with partition key." - - "Patch items with PatchInstructions and PatchOperation for partial updates." - - "Extracts partition key and ID from ResourceExt trait methods." - - "Uses TokenCredential for secure authentication." - - "Async/await with Tokio and proper Result-based error propagation." - - "Code compiles without warnings." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-cosmos-rust - graders: - - type: prompt - name: "Cosmos DB CRUD completeness" - config: - model: gpt-5.3-codex - prompt: | - - CosmosClient creation with builder and ProximityTo routing - - Database and container client navigation - - Create item with partition key and ID - - Read item by partition key - - Replace item for full updates - - Delete item by partition key - - Patch item with PatchInstructions - - PatchOperation::set() for field updates - - Response conversion with into_model() - - Error handling with ErrorResponse - - TokenCredential authentication - - Async/await with Tokio - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: CosmosClient Builder Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - CosmosClient created via CosmosClient::builder() - .build(account, RoutingStrategy::ProximityTo(region)).await? - Database accessed via client.database_client(name) - Container accessed via database.container_client(name).await - - name: CRUD Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Create: container.create_item(partition_key, id, item, None).await? - Read: container.read_item(partition_key, id, None).await? - Update: container.replace_item(partition_key, id, item, None).await? - Delete: container.delete_item(partition_key, id, None).await? - All require partition_key as first parameter - - name: Partition Key Management - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Partition key is required parameter in all item operations - Partition key value passed as string to operations - Document fields include partition_key field for consistency - Responses contain partition key information - - name: Patch Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Patch operations use PatchInstructions::from(vec![...]) - Operations created with PatchOperation::set(path, value) - Patched items obtained via container.patch_item(...).await? - Patch returns response with into_model() conversion - - name: Response Conversion and Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Responses converted with into_model() for deserialization - Error handling with match and error extraction - No .unwrap() on fallible operations - Result propagated through main with ? operator - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - All I/O operations awaited - Serde for serialization/deserialization - TokenCredential passed with Some(credential) diff --git a/tests/scenarios/azure-cosmos-rust/vally/eval.yaml b/tests/scenarios/azure-cosmos-rust/vally/eval.yaml index 097bc098..039aac5a 100644 --- a/tests/scenarios/azure-cosmos-rust/vally/eval.yaml +++ b/tests/scenarios/azure-cosmos-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-cosmos-crud-operations prompt: | @@ -62,7 +63,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper Result-based error propagation." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -75,6 +76,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-cosmos-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -84,7 +95,6 @@ stimuli: - type: prompt name: "Cosmos DB CRUD completeness" config: - model: gpt-5.3-codex prompt: | - CosmosClient creation with builder and ProximityTo routing - Database and container client navigation @@ -101,72 +111,68 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: CosmosClient Builder Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > CosmosClient created via CosmosClient::builder() .build(account, RoutingStrategy::ProximityTo(region)).await? @@ -175,7 +181,7 @@ stimuli: - name: CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > Create: container.create_item(partition_key, id, item, None).await? Read: container.read_item(partition_key, id, None).await? @@ -185,7 +191,7 @@ stimuli: - name: Partition Key Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > Partition key is required parameter in all item operations Partition key value passed as string to operations @@ -194,7 +200,7 @@ stimuli: - name: Patch Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > Patch operations use PatchInstructions::from(vec![...]) Operations created with PatchOperation::set(path, value) @@ -203,7 +209,7 @@ stimuli: - name: Response Conversion and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: gpt-5.5 prompt: > Responses converted with into_model() for deserialization Error handling with match and error extraction diff --git a/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml index da3af2ca..69239176 100644 --- a/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-cosmos-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-cosmos-crud-operations prompt: | @@ -62,7 +63,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper Result-based error propagation." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -90,7 +91,7 @@ stimuli: - type: prompt name: "Cosmos DB CRUD completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - CosmosClient creation with builder and ProximityTo routing - Database and container client navigation @@ -149,7 +150,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 2m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -172,7 +173,7 @@ stimuli: - name: CosmosClient Builder Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CosmosClient created via CosmosClient::builder() .build(account, RoutingStrategy::ProximityTo(region)).await? @@ -181,7 +182,7 @@ stimuli: - name: CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Create: container.create_item(partition_key, id, item, None).await? Read: container.read_item(partition_key, id, None).await? @@ -191,7 +192,7 @@ stimuli: - name: Partition Key Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Partition key is required parameter in all item operations Partition key value passed as string to operations @@ -200,7 +201,7 @@ stimuli: - name: Patch Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Patch operations use PatchInstructions::from(vec![...]) Operations created with PatchOperation::set(path, value) @@ -209,7 +210,7 @@ stimuli: - name: Response Conversion and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Responses converted with into_model() for deserialization Error handling with match and error extraction diff --git a/tests/scenarios/azure-data-tables-py/scenarios.yaml b/tests/scenarios/azure-data-tables-py/scenarios.yaml index a5d105bc..ab1ef824 100644 --- a/tests/scenarios/azure-data-tables-py/scenarios.yaml +++ b/tests/scenarios/azure-data-tables-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-data-tables-py/vally/eval.yaml b/tests/scenarios/azure-data-tables-py/vally/eval.yaml new file mode 100644 index 00000000..941b636a --- /dev/null +++ b/tests/scenarios/azure-data-tables-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-data-tables-py-eval +description: Vally evaluation suite for azure-data-tables-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_table_service_client + prompt: |- + Create a TableServiceClient using DefaultAzureCredential, create a table + if it doesn't exist, and get a TableClient for that table. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: table_client_entity_crud + prompt: |- + Create a TableClient and perform entity CRUD operations using + PartitionKey/RowKey, including update with UpdateMode. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-data-tables-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-data-tables-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..5737fda8 --- /dev/null +++ b/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-data-tables-py-eval +description: Vally evaluation suite for azure-data-tables-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_table_service_client + prompt: |- + Create a TableServiceClient using DefaultAzureCredential, create a table + if it doesn't exist, and get a TableClient for that table. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: table_client_entity_crud + prompt: |- + Create a TableClient and perform entity CRUD operations using + PartitionKey/RowKey, including update with UpdateMode. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-data-tables-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-data-tables-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..8d345bb7 --- /dev/null +++ b/tests/scenarios/azure-data-tables-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-data-tables-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-data-tables-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-eventgrid-py/scenarios.yaml b/tests/scenarios/azure-eventgrid-py/scenarios.yaml index 3477b107..0d60917a 100644 --- a/tests/scenarios/azure-eventgrid-py/scenarios.yaml +++ b/tests/scenarios/azure-eventgrid-py/scenarios.yaml @@ -7,7 +7,7 @@ leakspeed: "" description: "" config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-eventgrid-py/vally/eval.yaml b/tests/scenarios/azure-eventgrid-py/vally/eval.yaml new file mode 100644 index 00000000..da441e0f --- /dev/null +++ b/tests/scenarios/azure-eventgrid-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-eventgrid-py-eval +description: Vally evaluation suite for azure-eventgrid-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_cloudevent_publish + prompt: |- + Publish a single CloudEvent to an Event Grid custom topic. + Use DefaultAzureCredential, environment variables, and proper imports. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: batch_cloudevent_publish + prompt: |- + Send a batch of CloudEvents to Event Grid using a list. + Use DefaultAzureCredential and a topic endpoint from env. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-eventgrid-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventgrid-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..6c5a517c --- /dev/null +++ b/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-eventgrid-py-eval +description: Vally evaluation suite for azure-eventgrid-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_cloudevent_publish + prompt: |- + Publish a single CloudEvent to an Event Grid custom topic. + Use DefaultAzureCredential, environment variables, and proper imports. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: batch_cloudevent_publish + prompt: |- + Send a batch of CloudEvents to Event Grid using a list. + Use DefaultAzureCredential and a topic endpoint from env. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-eventgrid-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventgrid-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..0634c1b3 --- /dev/null +++ b/tests/scenarios/azure-eventgrid-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-eventgrid-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventgrid-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-eventhub-py/scenarios.yaml b/tests/scenarios/azure-eventhub-py/scenarios.yaml index d85da2a3..c375b5eb 100644 --- a/tests/scenarios/azure-eventhub-py/scenarios.yaml +++ b/tests/scenarios/azure-eventhub-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-eventhub-py/vally/eval.yaml b/tests/scenarios/azure-eventhub-py/vally/eval.yaml new file mode 100644 index 00000000..c74d3213 --- /dev/null +++ b/tests/scenarios/azure-eventhub-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-eventhub-py-eval +description: Vally evaluation suite for azure-eventhub-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: producer_batch_send + prompt: |- + Create a synchronous Event Hub producer that authenticates with + DefaultAzureCredential, creates a batch, handles batch size limits, + and sends the batch. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: consumer_receive_with_checkpoint + prompt: |- + Create an Event Hub consumer that uses DefaultAzureCredential and + receives events from the beginning of the stream while checkpointing + each event. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-eventhub-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventhub-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..8c48831e --- /dev/null +++ b/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-eventhub-py-eval +description: Vally evaluation suite for azure-eventhub-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: producer_batch_send + prompt: |- + Create a synchronous Event Hub producer that authenticates with + DefaultAzureCredential, creates a batch, handles batch size limits, + and sends the batch. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: consumer_receive_with_checkpoint + prompt: |- + Create an Event Hub consumer that uses DefaultAzureCredential and + receives events from the beginning of the stream while checkpointing + each event. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-eventhub-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventhub-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..cb6f64b4 --- /dev/null +++ b/tests/scenarios/azure-eventhub-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-eventhub-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-eventhub-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml deleted file mode 100644 index ef73d6e5..00000000 --- a/tests/scenarios/azure-eventhub-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,162 +0,0 @@ -name: azure-eventhub-rust-eval -description: Evaluation suite for azure-eventhub-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-eventhub-producer-consumer - prompt: | - Write a Rust program named rust-eventhub-demo that demonstrates producer and consumer patterns - for Azure Event Hubs: - - Producer operations: - 1. Create a ProducerClient with namespace credentials - 2. Send single events to the hub - 3. Create a batch and add multiple events to it - 4. Send the batch - 5. Handle backpressure and errors during sending - - Consumer operations: - 1. Create a ConsumerClient with the same credentials - 2. Open a receiver on partition "0" - 3. Set the start position to StartLocation::Earliest - 4. Stream events from the receiver - 5. Extract and display event body data - 6. Handle errors per event in the stream - - Program should demonstrate: - - DeveloperToolsCredential or ManagedIdentityCredential usage - - ProducerClient::builder() pattern for producer initialization - - ConsumerClient::builder() pattern for consumer initialization - - Single event sending via producer.send_event(vec![...], None).await - - Batch creation and event addition via create_batch() and try_add_event_data() - - Receiver creation on specific partition with start position configuration - - Stream processing with futures::stream::StreamExt and while let Some(...) pattern - - Proper async/await and error handling with Result types - - The program should compile and execute successfully, demonstrating the complete - event hub producer-consumer workflow with proper credential-based authentication. - - rubric: - - "Implements both ProducerClient and ConsumerClient from official azure_messaging_eventhubs SDK." - - "Uses TokenCredential-based authentication (DeveloperToolsCredential or ManagedIdentityCredential)." - - "ProducerClient sends single events and batch events to Event Hub." - - "ConsumerClient receives events from specific partition with configurable start position." - - "Demonstrates async/await Rust with Tokio runtime." - - "Event stream processing with proper error handling per event." - - "Code compiles with cargo check without warnings." - - "Builder pattern used for both client initialization." - - "StartLocation configuration for consumer start position (Earliest/Latest)." - - "Proper extraction of event body data from stream." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - - "**/examples/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - - "**/*.d" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-eventhub-rust - graders: - - type: prompt - name: "EventHub producer-consumer completeness" - config: - model: gpt-5.3-codex - prompt: | - - ProducerClient creation with builder pattern - - Single event sending: producer.send_event(vec![...], None).await - - Batch operations: create_batch() and try_add_event_data() - - ConsumerClient creation with builder pattern - - Receiver creation: open_receiver_on_partition() - - StartPosition configuration with StartLocation::Earliest - - Event stream processing with futures::stream::StreamExt - - Error handling for individual events in stream - - TokenCredential authentication - - Async/await patterns throughout - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - stderr_contains: "Finished" - - - name: ProducerClient Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ProducerClient created via ProducerClient::builder() - .open(namespace, eventhub_name, credential).await? - Sends single events via send_event(data, None).await - Creates batches via create_batch(None).await and adds events via try_add_event_data() - - name: ConsumerClient Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ConsumerClient created via ConsumerClient::builder() - .open(namespace, eventhub_name, credential).await? - Opens receiver on partition via open_receiver_on_partition() with partition ID as string - Sets StartPosition with StartLocation::Earliest or Latest - - name: Event Stream Processing - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Event stream obtained via receiver.stream_events() - Stream processing with while let Some(event_result) = stream.next().await - Match on Ok/Err for per-event error handling - Uses futures::stream::StreamExt trait for stream operations - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code uses #[tokio::main] attribute and async/await throughout - All I/O operations are properly awaited - Result> main signature - - name: Credential Management - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses DeveloperToolsCredential for local dev or ManagedIdentityCredential for production - Credential is cloned when passed to multiple clients - No hardcoded credentials in source code diff --git a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml index e1999661..5ba5449c 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-eventhub-producer-consumer prompt: | @@ -54,7 +55,7 @@ stimuli: - "Builder pattern used for both client initialization." - "StartLocation configuration for consumer start position (Earliest/Latest)." - "Proper extraction of event body data from stream." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -69,6 +70,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-eventhub-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -78,7 +89,6 @@ stimuli: - type: prompt name: "EventHub producer-consumer completeness" config: - model: gpt-5.3-codex prompt: | - ProducerClient creation with builder pattern - Single event sending: producer.send_event(vec![...], None).await @@ -93,73 +103,67 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s - stderr_contains: "Finished" + commands: + - command: cargo build + emit_in_metadata: true - name: ProducerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > ProducerClient created via ProducerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -168,7 +172,6 @@ stimuli: - name: ConsumerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > ConsumerClient created via ConsumerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -177,7 +180,6 @@ stimuli: - name: Event Stream Processing type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Event stream obtained via receiver.stream_events() Stream processing with while let Some(event_result) = stream.next().await @@ -186,7 +188,6 @@ stimuli: - name: Credential Management type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Uses DeveloperToolsCredential for local dev or ManagedIdentityCredential for production Credential is cloned when passed to multiple clients diff --git a/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml index 6ea37bd7..04589dd5 100644 --- a/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-eventhub-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-eventhub-producer-consumer prompt: | @@ -54,7 +55,7 @@ stimuli: - "Builder pattern used for both client initialization." - "StartLocation configuration for consumer start position (Earliest/Latest)." - "Proper extraction of event body data from stream." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -84,7 +85,7 @@ stimuli: - type: prompt name: "EventHub producer-consumer completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - ProducerClient creation with builder pattern - Single event sending: producer.send_event(vec![...], None).await @@ -165,7 +166,7 @@ stimuli: - name: ProducerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ProducerClient created via ProducerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -174,7 +175,7 @@ stimuli: - name: ConsumerClient Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ConsumerClient created via ConsumerClient::builder() .open(namespace, eventhub_name, credential).await? @@ -183,7 +184,7 @@ stimuli: - name: Event Stream Processing type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Event stream obtained via receiver.stream_events() Stream processing with while let Some(event_result) = stream.next().await @@ -192,7 +193,7 @@ stimuli: - name: Credential Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses DeveloperToolsCredential for local dev or ManagedIdentityCredential for production Credential is cloned when passed to multiple clients diff --git a/tests/scenarios/azure-identity-py/scenarios.yaml b/tests/scenarios/azure-identity-py/scenarios.yaml index 0e1289b5..13571309 100644 --- a/tests/scenarios/azure-identity-py/scenarios.yaml +++ b/tests/scenarios/azure-identity-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-identity-py/vally/eval.yaml b/tests/scenarios/azure-identity-py/vally/eval.yaml new file mode 100644 index 00000000..0586c33a --- /dev/null +++ b/tests/scenarios/azure-identity-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-identity-py-eval +description: Vally evaluation suite for azure-identity-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: default_credential_basic + prompt: |- + Show a minimal example using DefaultAzureCredential to request a token + for https://management.azure.com/.default. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: managed_identity_user_assigned + prompt: |- + Create a ManagedIdentityCredential using a user-assigned client ID from + AZURE_CLIENT_ID environment variable. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-identity-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-identity-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-identity-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-identity-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d8054b08 --- /dev/null +++ b/tests/scenarios/azure-identity-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-identity-py-eval +description: Vally evaluation suite for azure-identity-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: default_credential_basic + prompt: |- + Show a minimal example using DefaultAzureCredential to request a token + for https://management.azure.com/.default. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: managed_identity_user_assigned + prompt: |- + Create a ManagedIdentityCredential using a user-assigned client ID from + AZURE_CLIENT_ID environment variable. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential initialization and token acquisition in try/except blocks + * Handle: ClientAuthenticationError, CredentialUnavailableError, AzureError + * Example: try: credential = DefaultAzureCredential() except ClientAuthenticationError as e: print(f'Auth error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-identity-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-identity-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-identity-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-identity-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..0c55d9b6 --- /dev/null +++ b/tests/scenarios/azure-identity-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-identity-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-identity-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml deleted file mode 100644 index 7f12fdc5..00000000 --- a/tests/scenarios/azure-identity-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,204 +0,0 @@ -name: azure-identity-rust-eval -description: Evaluation suite for azure-identity-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-identity-credentials - prompt: | - Write a Rust program named rust-identity-demo that demonstrates all major credential - types from the Azure Identity SDK: - - Development credentials: - 1. Create a DeveloperToolsCredential with DeveloperToolsCredential::new(None)? - 2. Use it to authenticate (attempts Azure CLI, then Azure Developer CLI) - 3. Clone and pass to multiple Azure SDK clients - - Production credentials: - 1. Create a ManagedIdentityCredential with ManagedIdentityCredential::new(None)? - 2. Demonstrate system-assigned managed identity - 3. Show how to use with Azure-hosted resources (VMs, App Service, Functions, AKS) - - Service principal credentials: - 1. Create a ClientSecretCredential with tenant_id, client_id, and client_secret - 2. Demonstrate configuration from environment variables - 3. Show error handling for missing credentials - - Credential capabilities: - 1. Pass credentials to SecretClient or KeyClient constructors - 2. Clone credentials for use with multiple clients - 3. Handle credential errors (authentication failures, missing env vars) - - Demonstrate: - - DeveloperToolsCredential for local development - - ManagedIdentityCredential for Azure-hosted workloads - - ClientSecretCredential for CI/CD and service accounts - - Credential cloning for multi-client scenarios - - TokenCredential trait abstraction - - Environment variable usage (AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET) - - Error handling with Result-based patterns - - Proper async/await with Tokio runtime - - No hardcoded secrets - - The program should compile and execute successfully, demonstrating the full - credential landscape and how to choose credentials for different deployment scenarios. - - rubric: - - "Uses official azure_identity SDK for all credential types." - - "Demonstrates DeveloperToolsCredential for local development." - - "Demonstrates ManagedIdentityCredential for Azure-hosted resources." - - "Demonstrates ClientSecretCredential with tenant, client, and secret." - - "Credentials properly cloned when passing to multiple clients." - - "Environment variables used for sensitive configuration (never hardcoded)." - - "Credentials passed as Some(credential) to SDK clients." - - "TokenCredential trait used for credential abstraction." - - "Error handling for credential creation and missing env vars." - - "Documentation or comments explain when each credential type is appropriate." - - "Async/await with Tokio runtime." - - "Code compiles without warnings." - - "Result-based error propagation with ? operator." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-identity-rust - graders: - - type: prompt - name: "Identity credential completeness" - config: - model: gpt-5.3-codex - prompt: | - - DeveloperToolsCredential creation and usage - - ManagedIdentityCredential creation and usage - - ClientSecretCredential with tenant_id, client_id, client_secret - - Credential cloning for multiple clients - - TokenCredential trait for abstraction - - Environment variable usage (AZURE_CLIENT_ID, AZURE_TENANT_ID, AZURE_CLIENT_SECRET) - - Error handling for credential creation - - Documentation on credential selection per scenario - - Async/await with Tokio runtime - - No hardcoded secrets in source - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: DeveloperToolsCredential - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - DeveloperToolsCredential created via DeveloperToolsCredential::new(None)? - Attempts Azure CLI (az login) first, then Azure Developer CLI (azd auth login) - Perfect for local development; never used in production - - name: ManagedIdentityCredential - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ManagedIdentityCredential created via ManagedIdentityCredential::new(None)? - System-assigned managed identity for Azure-hosted workloads - Used in VMs, App Service, Functions, AKS, Container Instances - No environment variables needed; identity managed by Azure - - name: ClientSecretCredential - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ClientSecretCredential created with three parameters: - tenant_id (string), client_id (string), client_secret (string) - Typically sourced from environment variables to avoid hardcoding - Used for CI/CD pipelines and service principal authentication - - name: Credential Cloning - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Credentials are Arc-wrapped and support cloning - Pass credential.clone() to multiple client constructors - Single credential instance can authenticate multiple SDK clients - Reduces overhead and maintains consistent auth context - - name: TokenCredential Trait - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - All credential types implement TokenCredential trait - Allows generic code that accepts any credential type - SDK clients accept Some(credential) where credential: impl TokenCredential - - name: Environment Variable Configuration - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - AZURE_TENANT_ID: Tenant ID for service principal auth - AZURE_CLIENT_ID: Client/application ID - AZURE_CLIENT_SECRET: Client secret (never commit to source control) - std::env::var() used to read environment variables - - name: Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Credential creation returns Result - Missing or invalid credentials generate errors - Environment variable errors handled with match or ? - No unwrap() on credential creation - - name: Scenario Documentation - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code includes comments explaining when each credential is appropriate - DeveloperToolsCredential for local development - ManagedIdentityCredential for production on Azure - ClientSecretCredential for CI/CD and service accounts - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - Credential operations are async-compatible - Multiple clients can be created and used concurrently - Result propagated with ? operator diff --git a/tests/scenarios/azure-identity-rust/vally/eval.yaml b/tests/scenarios/azure-identity-rust/vally/eval.yaml index eacdf4f2..eaf46317 100644 --- a/tests/scenarios/azure-identity-rust/vally/eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-identity-credentials prompt: | @@ -63,7 +64,7 @@ stimuli: - "Async/await with Tokio runtime." - "Code compiles without warnings." - "Result-based error propagation with ? operator." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -76,6 +77,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-identity-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -85,7 +96,6 @@ stimuli: - type: prompt name: "Identity credential completeness" config: - model: gpt-5.3-codex prompt: | - DeveloperToolsCredential creation and usage - ManagedIdentityCredential creation and usage @@ -99,72 +109,67 @@ stimuli: - No hardcoded secrets in source scoring: scale_1_5 - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - name: DeveloperToolsCredential type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > DeveloperToolsCredential created via DeveloperToolsCredential::new(None)? Attempts Azure CLI (az login) first, then Azure Developer CLI (azd auth login) @@ -172,7 +177,6 @@ stimuli: - name: ManagedIdentityCredential type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > ManagedIdentityCredential created via ManagedIdentityCredential::new(None)? System-assigned managed identity for Azure-hosted workloads @@ -181,7 +185,6 @@ stimuli: - name: ClientSecretCredential type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > ClientSecretCredential created with three parameters: tenant_id (string), client_id (string), client_secret (string) @@ -190,7 +193,6 @@ stimuli: - name: Credential Cloning type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Credentials are Arc-wrapped and support cloning Pass credential.clone() to multiple client constructors @@ -199,7 +201,6 @@ stimuli: - name: TokenCredential Trait type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > All credential types implement TokenCredential trait Allows generic code that accepts any credential type @@ -207,7 +208,6 @@ stimuli: - name: Environment Variable Configuration type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > AZURE_TENANT_ID: Tenant ID for service principal auth AZURE_CLIENT_ID: Client/application ID @@ -216,7 +216,6 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Credential creation returns Result Missing or invalid credentials generate errors @@ -225,7 +224,6 @@ stimuli: - name: Scenario Documentation type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Code includes comments explaining when each credential is appropriate DeveloperToolsCredential for local development diff --git a/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml index 07b4383d..4195da52 100644 --- a/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-identity-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-identity-credentials prompt: | @@ -63,7 +64,7 @@ stimuli: - "Async/await with Tokio runtime." - "Code compiles without warnings." - "Result-based error propagation with ? operator." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -91,7 +92,7 @@ stimuli: - type: prompt name: "Identity credential completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - DeveloperToolsCredential creation and usage - ManagedIdentityCredential creation and usage @@ -148,7 +149,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -159,19 +160,19 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 3m - type: run-command name: "Cargo Build" config: command: cargo args: - "build" - timeout: 120s + timeout: 3m - name: DeveloperToolsCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > DeveloperToolsCredential created via DeveloperToolsCredential::new(None)? Attempts Azure CLI (az login) first, then Azure Developer CLI (azd auth login) @@ -179,7 +180,7 @@ stimuli: - name: ManagedIdentityCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ManagedIdentityCredential created via ManagedIdentityCredential::new(None)? System-assigned managed identity for Azure-hosted workloads @@ -188,7 +189,7 @@ stimuli: - name: ClientSecretCredential type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > ClientSecretCredential created with three parameters: tenant_id (string), client_id (string), client_secret (string) @@ -197,7 +198,7 @@ stimuli: - name: Credential Cloning type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Credentials are Arc-wrapped and support cloning Pass credential.clone() to multiple client constructors @@ -206,7 +207,7 @@ stimuli: - name: TokenCredential Trait type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > All credential types implement TokenCredential trait Allows generic code that accepts any credential type @@ -214,7 +215,7 @@ stimuli: - name: Environment Variable Configuration type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > AZURE_TENANT_ID: Tenant ID for service principal auth AZURE_CLIENT_ID: Client/application ID @@ -223,7 +224,7 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Credential creation returns Result Missing or invalid credentials generate errors @@ -232,7 +233,7 @@ stimuli: - name: Scenario Documentation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Code includes comments explaining when each credential is appropriate DeveloperToolsCredential for local development diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml deleted file mode 100644 index 811e9217..00000000 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,205 +0,0 @@ -name: azure-keyvault-certificates-rust-eval -description: Evaluation suite for azure-security-keyvault-certificates-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-keyvault-certificates-operations - prompt: | - Write a Rust program named rust-keyvault-certs that demonstrates X.509 certificate - management and long-running operations with Azure Key Vault: - - Setup: - 1. Create a CertificateClient with vault URL and TokenCredential - 2. Use DeveloperToolsCredential for local dev - - Certificate creation (Long-Running Operation): - 1. Define CertificatePolicy with X509CertificateProperties - 2. Set subject to "CN=example.com" - 3. Set IssuerParameters with name "Self" for self-signed - 4. Create certificate with begin_create_certificate() returning Poller - 5. Await the Poller for completion - 6. Extract certificate name and version via ResourceExt - - Certificate management: - 1. Get an existing certificate with get_certificate() - 2. Update certificate properties with UpdateCertificatePropertiesParameters - 3. Add tags to certificates - 4. Delete a certificate with delete_certificate() - 5. List certificates with list_certificate_properties() returning Pager - - Advanced operations: - 1. Use certificate key for signing via KeyClient integration - 2. Create SignParameters with digest and SignatureAlgorithm - 3. Demonstrate that certificates contain embedded keys - - Demonstrate: - - CertificateClient::new() with vault URL and credential - - CertificatePolicy with X509CertificateProperties and IssuerParameters - - CreateCertificateParameters with policy - - Poller from begin_create_certificate() as IntoFuture - - Direct .await on Poller for completion - - GetCertificate and UpdateCertificatePropertiesParameters - - Pager iteration with TryStreamExt::try_next() - - ResourceExt for name and version extraction - - Proper async/await with Tokio runtime - - Result-based error propagation - - The program should compile and execute successfully, demonstrating complete - certificate lifecycle including LRO handling and signing operations. - - rubric: - - "Uses official azure_security_keyvault_certificates SDK." - - "Creates CertificateClient with vault URL and TokenCredential." - - "Defines CertificatePolicy with X509CertificateProperties and subject." - - "Sets IssuerParameters with 'Self' for self-signed certificates." - - "Creates certificates with begin_create_certificate() returning Poller." - - "Awaits Poller directly for LRO completion." - - "Gets certificates with get_certificate() and converts with into_model()." - - "Updates certificate properties with tags." - - "Deletes certificates with delete_certificate()." - - "Lists certificates with pagination using list_certificate_properties()." - - "Uses ResourceExt trait to extract name and version." - - "Demonstrates certificate key access for signing operations." - - "Handles LRO errors gracefully." - - "Uses TokenCredential for secure authentication." - - "Async/await with Tokio and proper error handling." - - "Code compiles without warnings." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-keyvault-certificates-rust - graders: - - type: prompt - name: "Key Vault certificates completeness" - config: - model: gpt-5.3-codex - prompt: | - - CertificateClient creation with vault URL and credential - - CertificatePolicy with X509CertificateProperties - - Subject set to CN=example.com format - - IssuerParameters with "Self" for self-signed - - CreateCertificateParameters wrapping policy - - begin_create_certificate() returning Poller - - Direct .await on Poller (IntoFuture pattern) - - Get certificate and into_model() conversion - - Update certificate properties with tags - - Delete certificate operation - - List certificates with Pager pagination - - ResourceExt for name/version extraction - - TokenCredential authentication - - Async/await throughout - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: CertificateClient Creation - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - CertificateClient created via CertificateClient::new(vault_url, credential, None) - Vault URL formatted as https://vault_name.vault.azure.net/ - Credential is TokenCredential (DeveloperToolsCredential or ManagedIdentityCredential) - - name: Certificate Policy Definition - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - CertificatePolicy includes x509_certificate_properties and issuer_parameters - X509CertificateProperties has subject: Some("CN=example.com".into()) - IssuerParameters has name: Some("Self".into()) for self-signed certificates - - name: Long-Running Operation Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - begin_create_certificate() returns Poller - Poller implements IntoFuture trait - Direct await on Poller: client.begin_create_certificate(...)?.await? - Awaiting completes the LRO and returns the certificate - - name: Certificate Management Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Get: client.get_certificate(name, None).await?.into_model()? - Update: client.update_certificate_properties(name, params.try_into()?, None).await? - Delete: client.delete_certificate(name, None).await? - List: client.list_certificate_properties(None)? returns Pager - - name: Certificate Properties Update - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - UpdateCertificatePropertiesParameters with tags: Some(HashMap) - Tags stored as Vec of key-value pairs - Uses #[allow(clippy::needless_update)] with ..Default::default() - Properties persist across certificate versions - - name: Pagination and Resource Extraction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - List returns Pager, iterate with let mut pager = ...; while let Some(cert) = pager.try_next().await? - ResourceExt provides cert.resource_id()? with name and version - Uses TryStreamExt trait from futures crate - - name: Certificate Key Integration - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Certificates contain embedded keys for signing operations - KeyClient can sign with certificate name as key reference - Demonstrates that certificates are more than just X.509 data - - name: Async Runtime and Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - All I/O awaited including LRO completion - No .unwrap() on Result types - Proper error propagation with ? operator diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml index 37a4e92b..e2a28c6a 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-certificates-operations prompt: | @@ -71,7 +72,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -84,6 +85,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-keyvault-certificates-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -93,7 +104,6 @@ stimuli: - type: prompt name: "Key Vault certificates completeness" config: - model: gpt-5.3-codex prompt: | - CertificateClient creation with vault URL and credential - CertificatePolicy with X509CertificateProperties @@ -112,72 +122,67 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: CertificateClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > CertificateClient created via CertificateClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -185,7 +190,6 @@ stimuli: - name: Certificate Policy Definition type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > CertificatePolicy includes x509_certificate_properties and issuer_parameters X509CertificateProperties has subject: Some("CN=example.com".into()) @@ -193,7 +197,6 @@ stimuli: - name: Long-Running Operation Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > begin_create_certificate() returns Poller Poller implements IntoFuture trait @@ -202,7 +205,6 @@ stimuli: - name: Certificate Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Get: client.get_certificate(name, None).await?.into_model()? Update: client.update_certificate_properties(name, params.try_into()?, None).await? @@ -211,7 +213,6 @@ stimuli: - name: Certificate Properties Update type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > UpdateCertificatePropertiesParameters with tags: Some(HashMap) Tags stored as Vec of key-value pairs @@ -220,7 +221,6 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(cert) = pager.try_next().await? ResourceExt provides cert.resource_id()? with name and version @@ -228,7 +228,6 @@ stimuli: - name: Certificate Key Integration type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Certificates contain embedded keys for signing operations KeyClient can sign with certificate name as key reference @@ -236,7 +235,6 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Uses #[tokio::main] on main function All I/O awaited including LRO completion diff --git a/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml index 9ce9fcef..6b1de8c7 100644 --- a/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-certificates-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-certificates-operations prompt: | @@ -71,7 +72,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -99,7 +100,7 @@ stimuli: - type: prompt name: "Key Vault certificates completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - CertificateClient creation with vault URL and credential - CertificatePolicy with X509CertificateProperties @@ -126,6 +127,7 @@ stimuli: - "+nightly" - "-Zscript" - ".vally/tools/check-azure-crates.rs" + timeout: 3m - name: TokenCredential Authentication type: program weight: 1.0 @@ -160,7 +162,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -171,7 +173,7 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 3m - type: run-command name: "Cargo Build" config: @@ -183,7 +185,7 @@ stimuli: - name: CertificateClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CertificateClient created via CertificateClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -191,7 +193,7 @@ stimuli: - name: Certificate Policy Definition type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > CertificatePolicy includes x509_certificate_properties and issuer_parameters X509CertificateProperties has subject: Some("CN=example.com".into()) @@ -199,7 +201,7 @@ stimuli: - name: Long-Running Operation Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > begin_create_certificate() returns Poller Poller implements IntoFuture trait @@ -208,7 +210,7 @@ stimuli: - name: Certificate Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Get: client.get_certificate(name, None).await?.into_model()? Update: client.update_certificate_properties(name, params.try_into()?, None).await? @@ -217,7 +219,7 @@ stimuli: - name: Certificate Properties Update type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > UpdateCertificatePropertiesParameters with tags: Some(HashMap) Tags stored as Vec of key-value pairs @@ -226,7 +228,7 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(cert) = pager.try_next().await? ResourceExt provides cert.resource_id()? with name and version @@ -234,7 +236,7 @@ stimuli: - name: Certificate Key Integration type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Certificates contain embedded keys for signing operations KeyClient can sign with certificate name as key reference @@ -242,7 +244,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O awaited including LRO completion diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml deleted file mode 100644 index 11c078df..00000000 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,196 +0,0 @@ -name: azure-keyvault-keys-rust-eval -description: Evaluation suite for azure-security-keyvault-keys-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-keyvault-keys-operations - prompt: | - Write a Rust program named rust-keyvault-keys that demonstrates cryptographic key - management and envelope encryption with Azure Key Vault: - - Key management operations: - 1. Create a KeyClient with vault URL and TokenCredential - 2. Create EC and RSA keys with CreateKeyParameters - 3. Get a key with get_key() - 4. Update key properties with UpdateKeyPropertiesParameters and tags - 5. Delete a key with delete_key() - 6. List keys with list_key_properties() and pagination - - Cryptographic operations: - 1. Wrap a data encryption key (DEK) with a key encryption key (KEK) using wrap_key() - 2. Use EncryptionAlgorithm::RsaOaep256 for wrapping - 3. Unwrap encrypted keys with unwrap_key() using the saved key version - 4. Demonstrate envelope encryption pattern - - Key types demonstrated: - 1. EC keys with CurveName::P256 - 2. RSA keys with key_size: 2048 or 4096 - 3. HSM-protected variants (EcHsm, RsaHsm) - - Advanced patterns: - 1. Extract resource ID components using ResourceExt trait - 2. Retain key versions for unwrap operations - 3. Handle KeyOperationParameters for wrap/unwrap operations - 4. Iterate keys with Pager and TryStreamExt - - Demonstrate: - - KeyClient::new() with vault URL and credential - - CreateKeyParameters with KeyType and curve settings - - UpdateKeyPropertiesParameters with tags - - Wrap/unwrap pattern for envelope encryption - - KeyOperationParameters with algorithm and value - - ResourceExt for name/version extraction - - Proper async/await with Tokio runtime - - Result-based error propagation - - The program should compile and execute successfully, demonstrating complete - key lifecycle management and envelope encryption workflows. - - rubric: - - "Uses official azure_security_keyvault_keys SDK." - - "Creates KeyClient with vault URL and TokenCredential." - - "Creates EC keys with CurveName parameter." - - "Creates RSA keys with key_size parameter." - - "Gets keys with get_key() and converts with into_model()." - - "Updates key properties with tags." - - "Deletes keys with delete_key()." - - "Lists keys with pagination using list_key_properties()." - - "Wraps DEK with KEK using wrap_key() and EncryptionAlgorithm." - - "Unwraps encrypted keys with unwrap_key() using key version." - - "Preserves key versions for encryption/decryption pairs." - - "Uses ResourceExt trait to extract name and version." - - "Handles wrap/unwrap errors gracefully." - - "Uses TokenCredential for secure authentication." - - "Async/await with Tokio and proper error handling." - - "Code compiles without warnings." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-keyvault-keys-rust - graders: - - type: prompt - name: "Key Vault keys completeness" - config: - model: gpt-5.3-codex - prompt: | - - KeyClient creation with vault URL and credential - - Create EC keys with CurveName::P256 - - Create RSA keys with key_size 2048 or 4096 - - Get key operation and into_model() conversion - - Update key properties with tags - - Delete key operation - - List keys with Pager pagination - - Wrap operation with EncryptionAlgorithm::RsaOaep256 - - Unwrap operation preserving key versions - - Envelope encryption pattern demonstration - - ResourceExt for name/version extraction - - TokenCredential authentication - - Async/await throughout - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: KeyClient Creation - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - KeyClient created via KeyClient::new(vault_url, credential, None) - Vault URL formatted as https://vault_name.vault.azure.net/ - Credential is TokenCredential (DeveloperToolsCredential or ManagedIdentityCredential) - - name: Key Creation Patterns - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - EC keys: CreateKeyParameters with kty: Some(KeyType::Ec), curve: Some(CurveName::P256) - RSA keys: CreateKeyParameters with kty: Some(KeyType::Rsa), key_size: Some(2048) - Keys created via client.create_key(name, body.try_into()?, None).await? - - name: Key Management Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Get: client.get_key(name, None).await?.into_model()? - Update: client.update_key_properties(name, params.try_into()?, None).await? - Delete: client.delete_key(name, None).await? - List: client.list_key_properties(None)? returns Pager - - name: Envelope Encryption Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Wrap: client.wrap_key(key_name, params.try_into()?, None).await? - Parameters include algorithm: Some(EncryptionAlgorithm::RsaOaep256) and value: Some(dek) - Extract wrapped result and key version for later unwrap - Unwrap: client.unwrap_key(key_name, version, params.try_into()?, None).await? - Compare unwrapped value with original DEK - - name: Key Version Management - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Extract key version from response via resource_id()? - Version stored from wrap operation for later unwrap - Same version used to unwrap (immutable relationship) - Versions allow key rotation without losing decryption capability - - name: Pagination and Resource Extraction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - List returns Pager, iterate with let mut pager = ...; while let Some(key) = pager.try_next().await? - ResourceExt provides key.resource_id()? with name and version - Use TryStreamExt trait from futures crate - - name: Async Runtime and Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - All I/O awaited (client operations, wrapping, unwrapping) - No .unwrap() on Result types - Proper error propagation with ? operator diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml index b927c364..26ad503e 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-keys-operations prompt: | @@ -70,7 +71,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,6 +84,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-keyvault-keys-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -92,7 +103,6 @@ stimuli: - type: prompt name: "Key Vault keys completeness" config: - model: gpt-5.3-codex prompt: | - KeyClient creation with vault URL and credential - Create EC keys with CurveName::P256 @@ -110,72 +120,67 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: KeyClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > KeyClient created via KeyClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -183,7 +188,6 @@ stimuli: - name: Key Creation Patterns type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > EC keys: CreateKeyParameters with kty: Some(KeyType::Ec), curve: Some(CurveName::P256) RSA keys: CreateKeyParameters with kty: Some(KeyType::Rsa), key_size: Some(2048) @@ -191,7 +195,6 @@ stimuli: - name: Key Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Get: client.get_key(name, None).await?.into_model()? Update: client.update_key_properties(name, params.try_into()?, None).await? @@ -200,7 +203,6 @@ stimuli: - name: Envelope Encryption Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Wrap: client.wrap_key(key_name, params.try_into()?, None).await? Parameters include algorithm: Some(EncryptionAlgorithm::RsaOaep256) and value: Some(dek) @@ -210,7 +212,6 @@ stimuli: - name: Key Version Management type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Extract key version from response via resource_id()? Version stored from wrap operation for later unwrap @@ -219,7 +220,6 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(key) = pager.try_next().await? ResourceExt provides key.resource_id()? with name and version @@ -227,7 +227,6 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Uses #[tokio::main] on main function All I/O awaited (client operations, wrapping, unwrapping) diff --git a/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml index 3cdaaa30..3b2fd9b8 100644 --- a/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-keys-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-keys-operations prompt: | @@ -70,7 +71,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -98,7 +99,7 @@ stimuli: - type: prompt name: "Key Vault keys completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - KeyClient creation with vault URL and credential - Create EC keys with CurveName::P256 @@ -181,7 +182,7 @@ stimuli: - name: KeyClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > KeyClient created via KeyClient::new(vault_url, credential, None) Vault URL formatted as https://vault_name.vault.azure.net/ @@ -189,7 +190,7 @@ stimuli: - name: Key Creation Patterns type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > EC keys: CreateKeyParameters with kty: Some(KeyType::Ec), curve: Some(CurveName::P256) RSA keys: CreateKeyParameters with kty: Some(KeyType::Rsa), key_size: Some(2048) @@ -197,7 +198,7 @@ stimuli: - name: Key Management Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Get: client.get_key(name, None).await?.into_model()? Update: client.update_key_properties(name, params.try_into()?, None).await? @@ -206,7 +207,7 @@ stimuli: - name: Envelope Encryption Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Wrap: client.wrap_key(key_name, params.try_into()?, None).await? Parameters include algorithm: Some(EncryptionAlgorithm::RsaOaep256) and value: Some(dek) @@ -216,7 +217,7 @@ stimuli: - name: Key Version Management type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Extract key version from response via resource_id()? Version stored from wrap operation for later unwrap @@ -225,7 +226,7 @@ stimuli: - name: Pagination and Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List returns Pager, iterate with let mut pager = ...; while let Some(key) = pager.try_next().await? ResourceExt provides key.resource_id()? with name and version @@ -233,7 +234,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O awaited (client operations, wrapping, unwrapping) diff --git a/tests/scenarios/azure-keyvault-py/scenarios.yaml b/tests/scenarios/azure-keyvault-py/scenarios.yaml index 90d59178..89b9f1b5 100644 --- a/tests/scenarios/azure-keyvault-py/scenarios.yaml +++ b/tests/scenarios/azure-keyvault-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-keyvault-py/vally/eval.yaml b/tests/scenarios/azure-keyvault-py/vally/eval.yaml new file mode 100644 index 00000000..a34cd396 --- /dev/null +++ b/tests/scenarios/azure-keyvault-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-keyvault-py-eval +description: Vally evaluation suite for azure-keyvault-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: secret_crud_basic + prompt: |- + Create a basic Azure Key Vault secrets example that authenticates with + DefaultAzureCredential, sets a secret, gets it, then deletes and purges it. + Use AZURE_KEYVAULT_URL for the vault URL. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and secret/key/certificate operations in try/except blocks + * Handle: ResourceNotFoundError, HttpResponseError, ClientAuthenticationError, AzureError + * Example: try: secret = client.get_secret(name) except ResourceNotFoundError: print('Secret not found') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: list_secret_versions + prompt: |- + List all secret names and all versions for a specific secret using + SecretClient.list_properties_of_secrets and list_properties_of_secret_versions. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and secret/key/certificate operations in try/except blocks + * Handle: ResourceNotFoundError, HttpResponseError, ClientAuthenticationError, AzureError + * Example: try: secret = client.get_secret(name) except ResourceNotFoundError: print('Secret not found') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-keyvault-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-keyvault-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..a369f4b9 --- /dev/null +++ b/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-keyvault-py-eval +description: Vally evaluation suite for azure-keyvault-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: secret_crud_basic + prompt: |- + Create a basic Azure Key Vault secrets example that authenticates with + DefaultAzureCredential, sets a secret, gets it, then deletes and purges it. + Use AZURE_KEYVAULT_URL for the vault URL. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and secret/key/certificate operations in try/except blocks + * Handle: ResourceNotFoundError, HttpResponseError, ClientAuthenticationError, AzureError + * Example: try: secret = client.get_secret(name) except ResourceNotFoundError: print('Secret not found') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: list_secret_versions + prompt: |- + List all secret names and all versions for a specific secret using + SecretClient.list_properties_of_secrets and list_properties_of_secret_versions. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and secret/key/certificate operations in try/except blocks + * Handle: ResourceNotFoundError, HttpResponseError, ClientAuthenticationError, AzureError + * Example: try: secret = client.get_secret(name) except ResourceNotFoundError: print('Secret not found') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-keyvault-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-keyvault-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..706ceefa --- /dev/null +++ b/tests/scenarios/azure-keyvault-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-keyvault-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-keyvault-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml deleted file mode 100644 index 7be891f3..00000000 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,178 +0,0 @@ -name: azure-keyvault-secrets-rust-eval -description: Evaluation suite for azure-security-keyvault-secrets-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-keyvault-secrets-operations - prompt: | - Write a Rust program named rust-keyvault-secrets that demonstrates secure secret - management with Azure Key Vault: - - Setup: - 1. Create a SecretClient with vault URL and TokenCredential - 2. Use DeveloperToolsCredential for local dev - - CRUD operations: - 1. Set a secret with SetSecretParameters including value and content_type - 2. Get a secret with get_secret() - 3. Update secret properties with UpdateSecretPropertiesParameters - 4. Delete a secret with delete_secret() - 5. List secrets with list_secret_properties() returning Pager - - Advanced operations: - 1. Extract resource ID components (name, version) using ResourceExt trait - 2. Set content_type on secrets (e.g., "text/plain") - 3. Add tags to secrets using HashMap - 4. Handle pagination when listing secrets with try_next() from TryStreamExt - - Error handling: - 1. Match on Ok/Err for secret operations - 2. Extract ErrorResponse from errors - 3. Handle 404 errors for missing secrets gracefully - - Demonstrate: - - SecretClient::new() with vault URL and credential - - SetSecretParameters and try_into() for request conversion - - UpdateSecretPropertiesParameters with content_type and tags - - Pager iteration with TryStreamExt::try_next() - - ResourceExt trait for extracting name and version - - Proper async/await with Tokio runtime - - Result-based error propagation - - The program should compile and execute successfully, demonstrating complete - secret lifecycle management with proper credential-based authentication. - - rubric: - - "Uses official azure_security_keyvault_secrets SDK." - - "Creates SecretClient with vault URL and TokenCredential." - - "Sets secrets with SetSecretParameters including value and content_type." - - "Retrieves secrets with get_secret() and converts with into_model()." - - "Updates secret properties with tags and content_type." - - "Deletes secrets with delete_secret()." - - "Lists secrets returning Pager with pagination support." - - "Uses ResourceExt trait to extract name and version fields." - - "Iterates paginated results with TryStreamExt::try_next()." - - "Handles errors with match patterns and ErrorResponse extraction." - - "Uses TokenCredential for secure authentication." - - "Async/await with Tokio and proper error handling." - - "Code compiles without warnings." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-keyvault-secrets-rust - graders: - - type: prompt - name: "Key Vault secrets completeness" - config: - model: gpt-5.3-codex - prompt: | - - SecretClient creation with vault URL and credential - - Set secret with SetSecretParameters including value - - Get secret with into_model() conversion - - Update secret properties with tags and content_type - - Delete secret operation - - List secrets returning Pager - - Pagination with TryStreamExt::try_next() - - ResourceExt for name/version extraction - - Error matching and ErrorResponse handling - - TokenCredential authentication - - Async/await throughout - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: SecretClient Creation - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - SecretClient created via SecretClient::new(vault_url, credential, None) - Vault URL is vault_name.vault.azure.net in https protocol - Credential is TokenCredential (DeveloperToolsCredential or ManagedIdentityCredential) - - name: Secret CRUD Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Set: client.set_secret(name, params.try_into()?, None).await? - Get: client.get_secret(name, None).await?.into_model()? - Update: client.update_secret_properties(name, params.try_into()?, None).await? - Delete: client.delete_secret(name, None).await? - - name: SetSecretParameters Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - SetSecretParameters created with value: Some(value.into()) - Content type can be set via content_type: Some("text/plain".into()) - Uses #[allow(clippy::needless_update)] with ..Default::default() - - name: Listing with Pagination - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - List secrets returns Pager via client.list_secret_properties(None)? - Iterate with let mut pager = ...; while let Some(secret) = pager.try_next().await? - Uses futures::TryStreamExt trait for try_next() - Extract name/version via secret.resource_id()?.name - - name: Resource Extraction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Use ResourceExt trait via secret.resource_id()? - Returns ResourceId with name and version fields - Stored for audit or reconstruction purposes - - name: Async Runtime and Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] on main function - All I/O operations awaited - No .unwrap() on fallible operations - Result propagated with ? operator - Error extraction with into_inner()? on error types diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml index 632382dd..866150c1 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-secrets-operations prompt: | @@ -63,7 +64,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -76,6 +77,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-keyvault-secrets-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -85,7 +96,6 @@ stimuli: - type: prompt name: "Key Vault secrets completeness" config: - model: gpt-5.3-codex prompt: | - SecretClient creation with vault URL and credential - Set secret with SetSecretParameters including value @@ -100,72 +110,67 @@ stimuli: - Async/await throughout scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - - type: run-command - name: "Cargo Check" + - name: "Cargo Check" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true + - name: "Cargo Build" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s + commands: + - command: cargo build + emit_in_metadata: true - name: SecretClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > SecretClient created via SecretClient::new(vault_url, credential, None) Vault URL is vault_name.vault.azure.net in https protocol @@ -173,7 +178,6 @@ stimuli: - name: Secret CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Set: client.set_secret(name, params.try_into()?, None).await? Get: client.get_secret(name, None).await?.into_model()? @@ -182,7 +186,6 @@ stimuli: - name: SetSecretParameters Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > SetSecretParameters created with value: Some(value.into()) Content type can be set via content_type: Some("text/plain".into()) @@ -190,7 +193,6 @@ stimuli: - name: Listing with Pagination type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > List secrets returns Pager via client.list_secret_properties(None)? Iterate with let mut pager = ...; while let Some(secret) = pager.try_next().await? @@ -199,7 +201,6 @@ stimuli: - name: Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Use ResourceExt trait via secret.resource_id()? Returns ResourceId with name and version fields @@ -207,7 +208,6 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Uses #[tokio::main] on main function All I/O operations awaited diff --git a/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml index d12448ac..1278b75a 100644 --- a/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-keyvault-secrets-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-keyvault-secrets-operations prompt: | @@ -63,7 +64,7 @@ stimuli: - "Uses TokenCredential for secure authentication." - "Async/await with Tokio and proper error handling." - "Code compiles without warnings." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -91,7 +92,7 @@ stimuli: - type: prompt name: "Key Vault secrets completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - SecretClient creation with vault URL and credential - Set secret with SetSecretParameters including value @@ -149,7 +150,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 180s - name: Rust Clippy Lints type: run-command weight: 1.0 @@ -160,19 +161,19 @@ stimuli: - "--" - "-D" - "warnings" - timeout: 120s + timeout: 300s - type: run-command name: "Cargo Build" config: command: cargo args: - "build" - timeout: 120s + timeout: 300s - name: SecretClient Creation type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > SecretClient created via SecretClient::new(vault_url, credential, None) Vault URL is vault_name.vault.azure.net in https protocol @@ -180,7 +181,7 @@ stimuli: - name: Secret CRUD Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Set: client.set_secret(name, params.try_into()?, None).await? Get: client.get_secret(name, None).await?.into_model()? @@ -189,7 +190,7 @@ stimuli: - name: SetSecretParameters Pattern type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > SetSecretParameters created with value: Some(value.into()) Content type can be set via content_type: Some("text/plain".into()) @@ -197,7 +198,7 @@ stimuli: - name: Listing with Pagination type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > List secrets returns Pager via client.list_secret_properties(None)? Iterate with let mut pager = ...; while let Some(secret) = pager.try_next().await? @@ -206,7 +207,7 @@ stimuli: - name: Resource Extraction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Use ResourceExt trait via secret.resource_id()? Returns ResourceId with name and version fields @@ -214,7 +215,7 @@ stimuli: - name: Async Runtime and Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Uses #[tokio::main] on main function All I/O operations awaited diff --git a/tests/scenarios/azure-messaging-webpubsubservice-py/scenarios.yaml b/tests/scenarios/azure-messaging-webpubsubservice-py/scenarios.yaml index 8a373689..853acd17 100644 --- a/tests/scenarios/azure-messaging-webpubsubservice-py/scenarios.yaml +++ b/tests/scenarios/azure-messaging-webpubsubservice-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-messaging-webpubsubservice-py/vally/eval.yaml b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/eval.yaml new file mode 100644 index 00000000..85fcd9c1 --- /dev/null +++ b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-messaging-webpubsubservice-py-eval +description: Vally evaluation suite for azure-messaging-webpubsubservice-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: connection_string_broadcast + prompt: |- + Create a WebPubSubServiceClient using a connection string from environment + variables, then send a text and JSON message to all connections. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: default_credential_targeted_sends + prompt: |- + Use DefaultAzureCredential with endpoint and hub from environment variables, + then send a message to a user and to a group. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-messaging-webpubsubservice-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..b54aee1f --- /dev/null +++ b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-messaging-webpubsubservice-py-eval +description: Vally evaluation suite for azure-messaging-webpubsubservice-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: connection_string_broadcast + prompt: |- + Create a WebPubSubServiceClient using a connection string from environment + variables, then send a text and JSON message to all connections. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: default_credential_targeted_sends + prompt: |- + Use DefaultAzureCredential with endpoint and hub from environment variables, + then send a message to a user and to a group. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-messaging-webpubsubservice-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..dade853e --- /dev/null +++ b/tests/scenarios/azure-messaging-webpubsubservice-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-messaging-webpubsubservice-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-messaging-webpubsubservice-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-mgmt-apicenter-py/scenarios.yaml b/tests/scenarios/azure-mgmt-apicenter-py/scenarios.yaml index 2a542cbf..2ff287af 100644 --- a/tests/scenarios/azure-mgmt-apicenter-py/scenarios.yaml +++ b/tests/scenarios/azure-mgmt-apicenter-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 @@ -23,7 +23,7 @@ scenarios: forbidden_patterns: - "hardcoded" - "xxxxx" - - "subscription_id=\"" + - 'subscription_id="' - "credential = Cli" tags: - basic @@ -34,22 +34,22 @@ scenarios: from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient from azure.mgmt.apicenter.models import Service - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + service = client.services.create_or_update( resource_group_name=resource_group, service_name="my-api-center", resource=Service(location="eastus"), ) - + print(f"Created service: {service.name}") # API Registration Workflow @@ -62,7 +62,7 @@ scenarios: - "create_or_update" - "ApiKind.REST" - "LifecycleStage" - - "workspace_name=\"default\"" + - 'workspace_name="default"' - "Api(" - "kind=" - "state=" @@ -80,16 +80,16 @@ scenarios: from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient from azure.mgmt.apicenter.models import Api, ApiKind, LifecycleStage - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + api = client.apis.create_or_update( resource_group_name=resource_group, service_name="my-api-center", @@ -102,7 +102,7 @@ scenarios: summary="API for managing pet store", ), ) - + print(f"Registered API: {api.title}") # API Version and Definition @@ -116,11 +116,12 @@ scenarios: - "api_definitions.create_or_update" - "ApiVersion(" - "ApiDefinition(" - - "specification=" - - "VersionStage" + - "ApiVersionProperties(" + - "ApiDefinitionProperties(" + - "begin_import_specification" - "workspace_name=" forbidden_patterns: - - "import_specification" + # - "import_specification" # Should be forbidden because the actual API is `begin_import_specification` - "swagger_url=" - "workspace_name=None" - "version_name=None" @@ -130,54 +131,68 @@ scenarios: - governance mock_response: | import os + from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient from azure.mgmt.apicenter.models import ( - ApiVersion, + ApiSpecImportRequest, + ApiSpecImportSourceFormat, ApiDefinition, - VersionStage, - ExternalDocumentationProperties, + ApiDefinitionProperties, + ApiVersion, + ApiVersionProperties, + LifecycleStage, ) - - subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") - resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - - credential = DefaultAzureCredential() - client = ApiCenterMgmtClient( - subscription_id=subscription_id, + + credential = DefaultAzureCredential(require_envvar=True) + + with ApiCenterMgmtClient( credential=credential, - ) - - # Create API version - version = client.api_versions.create_or_update( - resource_group_name=resource_group, - service_name="my-api-center", - workspace_name="default", - api_name="pet-api", - version_name="v1.0", - resource=ApiVersion( - title="Version 1.0", - stage=VersionStage.PRODUCTION, - ), - ) - - # Create API definition with spec - definition = client.api_definitions.create_or_update( - resource_group_name=resource_group, - service_name="my-api-center", - workspace_name="default", - api_name="pet-api", - version_name="v1.0", - definition_name="openapi", - resource=ApiDefinition( - title="OpenAPI Definition", - specification=ExternalDocumentationProperties( - url="https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/examples/v3.0/petstore.json" + subscription_id=os.environ["AZURE_SUBSCRIPTION_ID"], + ) as client: + version = client.api_versions.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1.0", + resource=ApiVersion( + properties=ApiVersionProperties( + title="Version 1.0", + lifecycle_stage=LifecycleStage.PRODUCTION, + ), ), - ), - ) - - print(f"Created version: {version.title}, Definition: {definition.title}") + ) + + client.api_definitions.create_or_update( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1.0", + definition_name="openapi", + resource=ApiDefinition( + properties=ApiDefinitionProperties( + title="OpenAPI Definition", + description="OpenAPI 3.0 specification", + ), + ), + ) + + client.api_definitions.begin_import_specification( + resource_group_name="my-resource-group", + service_name="my-api-center", + workspace_name="default", + api_name="my-api", + version_name="v1.0", + definition_name="openapi", + body=ApiSpecImportRequest( + format=ApiSpecImportSourceFormat.INLINE, + value='{"openapi":"3.0.0","info":{"title":"My API","version":"1.0","description":"Production OpenAPI specification for My API"},"paths":{}}', + ), + ).result() + + print(f"Created API version and imported specification: {version.title}") # Environment and Deployment Tracking - name: environment_and_deployment @@ -194,7 +209,6 @@ scenarios: - "deployment_name=" - "workspace_name=" forbidden_patterns: - - "environment_id=" - "deployment_id=" - "workspace_name=None" - "environment_name=None" @@ -204,55 +218,64 @@ scenarios: - tracking - governance mock_response: | - import os + from os import environ + from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient - from azure.mgmt.apicenter.models import ( - Environment, - Deployment, - DeploymentState, - EnvironmentKind, - ) - - subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") - resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - - credential = DefaultAzureCredential() - client = ApiCenterMgmtClient( - subscription_id=subscription_id, + from azure.mgmt.apicenter.models import Deployment, DeploymentState, Environment, EnvironmentKind + + RESOURCE_GROUP_NAME = "my-resource-group" + SERVICE_NAME = "my-api-center" + WORKSPACE_NAME = "default" + API_NAME = "my-api" + VERSION_NAME = "v1" + DEFINITION_NAME = "openapi" + ENVIRONMENT_NAME = "production" + DEPLOYMENT_NAME = "prod-deployment" + + credential = DefaultAzureCredential(require_envvar=True) + + with ApiCenterMgmtClient( credential=credential, - ) - - # Create environment - environment = client.environments.create_or_update( - resource_group_name=resource_group, - service_name="my-api-center", - workspace_name="default", - environment_name="production", - resource=Environment( - kind=EnvironmentKind.PRODUCTION, - title="Production", - server={"url": "https://api.example.com"}, - ), - ) - - # Track deployment - deployment = client.deployments.create_or_update( - resource_group_name=resource_group, - service_name="my-api-center", - workspace_name="default", - deployment_name="prod-2024-01", - resource=Deployment( - environment_name="production", - api_name="pet-api", - api_version="v1.0", - definition_name="openapi", - state=DeploymentState.ACTIVE, - title="Production Deployment Jan 2024", - ), - ) - - print(f"Environment: {environment.title}, Deployment: {deployment.title}") + subscription_id=environ["AZURE_SUBSCRIPTION_ID"], + ) as client: + environment = client.environments.create_or_update( + resource_group_name=RESOURCE_GROUP_NAME, + service_name=SERVICE_NAME, + workspace_name=WORKSPACE_NAME, + environment_name=ENVIRONMENT_NAME, + resource=Environment( + title="Production", + description="Production environment", + kind=EnvironmentKind.PRODUCTION, + server={ + "type": "Azure API Management", + "management_portal_uri": ["https://portal.azure.com"], + }, + ), + ) + + deployment = client.deployments.create_or_update( + resource_group_name=RESOURCE_GROUP_NAME, + service_name=SERVICE_NAME, + workspace_name=WORKSPACE_NAME, + api_name=API_NAME, + deployment_name=DEPLOYMENT_NAME, + resource=Deployment( + title="Production Deployment", + description="Deployed to production API Management", + environment_id=f"/workspaces/{WORKSPACE_NAME}/environments/{ENVIRONMENT_NAME}", + definition_id=( + f"/workspaces/{WORKSPACE_NAME}/apis/{API_NAME}" + f"/versions/{VERSION_NAME}/definitions/{DEFINITION_NAME}" + ), + state=DeploymentState.ACTIVE, + server={"runtime_uri": ["https://api.example.com"]}, + ), + ) + + print(f"Environment tracked: {environment.name} ({environment.kind})") + print(f"Deployment tracked: {deployment.name} ({deployment.state})") # Metadata Governance - Define Custom Schema - name: metadata_governance_schema @@ -284,16 +307,16 @@ scenarios: MetadataSchema, MetadataSchemaProperty, ) - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + # Define custom metadata schema schema = client.metadata_schemas.create_or_update( resource_group_name=resource_group, @@ -317,7 +340,7 @@ scenarios: ] ), ) - + print(f"Created metadata schema: {schema.name}") # List and Search APIs with Filtering @@ -344,23 +367,23 @@ scenarios: import os from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + # List all APIs in workspace apis = client.apis.list( resource_group_name=resource_group, service_name="my-api-center", workspace_name="default", ) - + print("APIs in catalog:") for api in apis: print(f" - {api.name}: {api.title}") @@ -393,16 +416,16 @@ scenarios: from azure.identity import DefaultAzureCredential from azure.mgmt.apicenter import ApiCenterMgmtClient from azure.mgmt.apicenter.models import Api, LifecycleStage - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + # Get existing API api = client.apis.get( resource_group_name=resource_group, @@ -410,7 +433,7 @@ scenarios: workspace_name="default", api_name="old-api", ) - + # Update lifecycle stage to deprecated updated_api = client.apis.create_or_update( resource_group_name=resource_group, @@ -424,7 +447,7 @@ scenarios: summary="This API is deprecated. Use new-api instead.", ), ) - + print(f"Updated {api.title} to {updated_api.state}") # Complete Governance Workflow @@ -451,7 +474,7 @@ scenarios: - "hardcoded" - "xxxxx" - "workspace_name=None" - - "subscription_id=\"" + - 'subscription_id="' tags: - workflow - governance @@ -472,16 +495,16 @@ scenarios: VersionStage, ExternalDocumentationProperties, ) - + subscription_id = os.environ.get("AZURE_SUBSCRIPTION_ID") resource_group = os.environ.get("AZURE_RESOURCE_GROUP") - + credential = DefaultAzureCredential() client = ApiCenterMgmtClient( subscription_id=subscription_id, credential=credential, ) - + # 1. Create API Center service service = client.services.create_or_update( resource_group_name=resource_group, @@ -489,7 +512,7 @@ scenarios: resource=Service(location="eastus"), ) print(f"1. Created service: {service.name}") - + # 2. Register API api = client.apis.create_or_update( resource_group_name=resource_group, @@ -504,7 +527,7 @@ scenarios: ), ) print(f"2. Registered API: {api.title}") - + # 3. Create version and spec version = client.api_versions.create_or_update( resource_group_name=resource_group, @@ -518,7 +541,7 @@ scenarios: ), ) print(f"3. Created version: {version.title}") - + # 4. Define custom governance schema schema = client.metadata_schemas.create_or_update( resource_group_name=resource_group, @@ -541,7 +564,7 @@ scenarios: ), ) print(f"4. Created governance schema: {schema.name}") - + # 5. List all APIs print("5. Registered APIs:") apis = client.apis.list( diff --git a/tests/scenarios/azure-mgmt-apicenter-py/vally/eval.yaml b/tests/scenarios/azure-mgmt-apicenter-py/vally/eval.yaml new file mode 100644 index 00000000..f06876a5 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apicenter-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-mgmt-apicenter-py-eval +description: Vally evaluation suite for azure-mgmt-apicenter-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_api_center_creation + prompt: |- + Create a basic Azure API Center service with proper authentication, + specify a location (eastus), and include resource cleanup. + Use environment variables for configuration. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: api_registration_workflow + prompt: |- + Register a REST API in the API Center with metadata. + Include the API kind (REST), lifecycle stage, and description. + Use the "default" workspace for API operations. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-apicenter-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..4c5e351a --- /dev/null +++ b/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-mgmt-apicenter-py-eval +description: Vally evaluation suite for azure-mgmt-apicenter-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_api_center_creation + prompt: |- + Create a basic Azure API Center service with proper authentication, + specify a location (eastus), and include resource cleanup. + Use environment variables for configuration. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: api_registration_workflow + prompt: |- + Register a REST API in the API Center with metadata. + Include the API kind (REST), lifecycle stage, and description. + Use the "default" workspace for API operations. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-apicenter-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..03f76966 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apicenter-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-mgmt-apicenter-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apicenter-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-mgmt-apimanagement-py/scenarios.yaml b/tests/scenarios/azure-mgmt-apimanagement-py/scenarios.yaml index c3c106dc..dc976ccd 100644 --- a/tests/scenarios/azure-mgmt-apimanagement-py/scenarios.yaml +++ b/tests/scenarios/azure-mgmt-apimanagement-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-mgmt-apimanagement-py/vally/eval.yaml b/tests/scenarios/azure-mgmt-apimanagement-py/vally/eval.yaml new file mode 100644 index 00000000..7a607fe1 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apimanagement-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-mgmt-apimanagement-py-eval +description: Vally evaluation suite for azure-mgmt-apimanagement-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_apim_client_creation + prompt: |- + Create a basic Azure API Management client with proper authentication + and environment variable handling. Show how to connect to an APIM service. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: apim_service_creation_with_sku + prompt: |- + Create an Azure API Management service with Developer SKU. + Include publisher email and name. Handle the long-running operation properly. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-apimanagement-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..fba45a44 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-mgmt-apimanagement-py-eval +description: Vally evaluation suite for azure-mgmt-apimanagement-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_apim_client_creation + prompt: |- + Create a basic Azure API Management client with proper authentication + and environment variable handling. Show how to connect to an APIM service. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: apim_service_creation_with_sku + prompt: |- + Create an Azure API Management service with Developer SKU. + Include publisher email and name. Handle the long-running operation properly. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-apimanagement-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..0dfe0d21 --- /dev/null +++ b/tests/scenarios/azure-mgmt-apimanagement-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-mgmt-apimanagement-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-apimanagement-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-mgmt-botservice-py/scenarios.yaml b/tests/scenarios/azure-mgmt-botservice-py/scenarios.yaml index 70034961..12f44a57 100644 --- a/tests/scenarios/azure-mgmt-botservice-py/scenarios.yaml +++ b/tests/scenarios/azure-mgmt-botservice-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-mgmt-botservice-py/vally/eval.yaml b/tests/scenarios/azure-mgmt-botservice-py/vally/eval.yaml new file mode 100644 index 00000000..1b8db751 --- /dev/null +++ b/tests/scenarios/azure-mgmt-botservice-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-mgmt-botservice-py-eval +description: Vally evaluation suite for azure-mgmt-botservice-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_bot_creation + prompt: |- + Create a basic Azure Bot Service using the management SDK. + Include proper authentication, bot creation with F0 SKU, and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: bot_retrieval_operations + prompt: |- + Write code to retrieve bot information from Azure Bot Service. + Include getting a specific bot, listing bots in a resource group, + and accessing bot properties. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-botservice-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..9a8ce29a --- /dev/null +++ b/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-mgmt-botservice-py-eval +description: Vally evaluation suite for azure-mgmt-botservice-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_bot_creation + prompt: |- + Create a basic Azure Bot Service using the management SDK. + Include proper authentication, bot creation with F0 SKU, and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: bot_retrieval_operations + prompt: |- + Write code to retrieve bot information from Azure Bot Service. + Include getting a specific bot, listing bots in a resource group, + and accessing bot properties. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-botservice-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..6deaacba --- /dev/null +++ b/tests/scenarios/azure-mgmt-botservice-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-mgmt-botservice-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-botservice-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-mgmt-fabric-py/scenarios.yaml b/tests/scenarios/azure-mgmt-fabric-py/scenarios.yaml index 97f10fbc..24737b6b 100644 --- a/tests/scenarios/azure-mgmt-fabric-py/scenarios.yaml +++ b/tests/scenarios/azure-mgmt-fabric-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-mgmt-fabric-py/vally/eval.yaml b/tests/scenarios/azure-mgmt-fabric-py/vally/eval.yaml new file mode 100644 index 00000000..e67d51c7 --- /dev/null +++ b/tests/scenarios/azure-mgmt-fabric-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-mgmt-fabric-py-eval +description: Vally evaluation suite for azure-mgmt-fabric-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_creation + prompt: |- + Create a basic Azure Fabric Management client that connects to Azure + using DefaultAzureCredential. Include proper authentication and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: create_fabric_capacity + prompt: |- + Create a new Fabric capacity with the Azure Fabric Management SDK. + Include the SKU configuration, location, and proper LRO handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-fabric-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..218f4f22 --- /dev/null +++ b/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-mgmt-fabric-py-eval +description: Vally evaluation suite for azure-mgmt-fabric-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_creation + prompt: |- + Create a basic Azure Fabric Management client that connects to Azure + using DefaultAzureCredential. Include proper authentication and cleanup. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: create_fabric_capacity + prompt: |- + Create a new Fabric capacity with the Azure Fabric Management SDK. + Include the SKU configuration, location, and proper LRO handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-mgmt-fabric-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..caf6dfda --- /dev/null +++ b/tests/scenarios/azure-mgmt-fabric-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-mgmt-fabric-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-mgmt-fabric-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-monitor-ingestion-py/scenarios.yaml b/tests/scenarios/azure-monitor-ingestion-py/scenarios.yaml index d6cef585..e7ff7431 100644 --- a/tests/scenarios/azure-monitor-ingestion-py/scenarios.yaml +++ b/tests/scenarios/azure-monitor-ingestion-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/eval.yaml b/tests/scenarios/azure-monitor-ingestion-py/vally/eval.yaml new file mode 100644 index 00000000..7c8add38 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/eval.yaml @@ -0,0 +1,138 @@ +name: azure-monitor-ingestion-py-eval +description: Vally evaluation suite for azure-monitor-ingestion-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: sync_basic_upload + prompt: |- + Provide a minimal synchronous example that uploads custom logs. + Use DefaultAzureCredential, a DCE endpoint from AZURE_DCE_ENDPOINT, + and DCR rule/stream names from AZURE_DCR_RULE_ID and AZURE_DCR_STREAM_NAME. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: async_upload_with_context_manager + prompt: |- + Provide an async example using azure.monitor.ingestion.aio LogsIngestionClient. + Use async with and await upload. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-ingestion-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/full-output.txt b/tests/scenarios/azure-monitor-ingestion-py/vally/full-output.txt new file mode 100644 index 00000000..7de81cf1 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/full-output.txt @@ -0,0 +1,94 @@ +Running experiment 'azure-monitor-ingestion-py-skill-experiment' +Output: Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-35-17-490Z + +Loaded 4 stimuli from 2 eval(s) +Running with 5 workers + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ sync_basic_upload [variant: sonnet_baseline] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 29,139 + Turns 2 + Tool calls 2 + Wall time 27.4s + Errors 0 + Skills used 0 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The generated code is excellent across all rubric criteria. It uses the correct Azure SDK APIs with proper imports, reads all configuration from environment variables, provides complete executable code with a main() function and __main__ guard, handles all required exception types explicitly with no bare excepts, and follows modern Python conventions throughout. The conditional import pattern for ServiceError and ValidationError is a practical touch that handles SDK version differences. The code would be immediately runnable with the proper Azure environment configured. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ async_upload_with_context_manager [variant: sonnet_baseline] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 29,267 + Turns 2 + Tool calls 2 + Wall time 29.5s + Errors 0 + Skills used 0 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The generated code is excellent. It correctly uses the async Azure Monitor Logs Ingestion client with the exact patterns requested (async with, await upload), handles all four required exception types explicitly with no bare excepts, avoids hardcoded secrets by using DefaultAzureCredential, follows modern Python conventions throughout, and produces a complete, runnable file. The only minor point is that ENDPOINT and RULE_ID could be read from os.environ rather than being placeholder strings, but this is a common and acceptable pattern in Azure SDK examples. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ sync_basic_upload [variant: sonnet_skill] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 47,855 + Turns 3 + Tool calls 3 + Wall time 28.9s + Errors 0 + Skills used 1 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The solution is solid and largely idiomatic: it creates a runnable main.py, uses the correct SDK APIs and workflow, keeps all config in environment variables, uses a context manager for client lifecycle, and avoids bare excepts and wildcard imports. The main deductions are (1) the import of LogsUploadError from a private module (_exceptions), which is fragile across SDK versions, and (2) some explicitly requested exception types (ServiceError, ValidationError, AzureError) are not handled by nameΓÇöonly covered by the catch-all. These are minor but real gaps against the stated requirements. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ async_upload_with_context_manager [variant: sonnet_skill] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 48,624 + Turns 3 + Tool calls 3 + Wall time 33.2s + Errors 0 + Skills used 1 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The code is a solid, nearly complete async ingestion example that correctly demonstrates the Azure Monitor Ingestion async client with context managers, credential handling via environment variables, and a retry pattern for failed logs. It loses points mainly because `ValidationError` is entirely absent and `ServiceError` is not handled as required by the task's critical exception-handling specification. The private `_models` import is a minor style issue. Everything else ΓÇö structure, naming, control flow, and use of the correct async APIs ΓÇö is well-executed. + + All graders passed. + + Γ£ö azure-monitor-ingestion-py-eval [claude-sonnet-4.6] score: 91.7% (threshold: 75.0%) + + Γ£ö azure-monitor-ingestion-py-eval [claude-sonnet-4.6] score: 100.0% (threshold: 75.0%) + +Saved artifacts +Markdown ΓåÆ Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-35-17-490Z\report.md + +Γ£ö sonnet_skill: 1/1 evals passed +Γ£ö sonnet_baseline: 1/1 evals passed + +Run ID: 66a90c8e-0ddd-4cf9-a4b1-9b82fc1af30f +Output: Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-35-17-490Z diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..920cb145 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-monitor-ingestion-py-eval +description: Vally evaluation suite for azure-monitor-ingestion-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: sync_basic_upload + prompt: |- + Provide a minimal synchronous example that uploads custom logs. + Use DefaultAzureCredential, a DCE endpoint from AZURE_DCE_ENDPOINT, + and DCR rule/stream names from AZURE_DCR_RULE_ID and AZURE_DCR_STREAM_NAME. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: async_upload_with_context_manager + prompt: |- + Provide an async example using azure.monitor.ingestion.aio LogsIngestionClient. + Use async with and await upload. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-ingestion-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..bf835072 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-monitor-ingestion-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-ingestion-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-monitor-ingestion-py/vally/vally-output.txt b/tests/scenarios/azure-monitor-ingestion-py/vally/vally-output.txt new file mode 100644 index 00000000..56c291f4 --- /dev/null +++ b/tests/scenarios/azure-monitor-ingestion-py/vally/vally-output.txt @@ -0,0 +1,94 @@ +Running experiment 'azure-monitor-ingestion-py-skill-experiment' +Output: Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-40-22-287Z + +Loaded 4 stimuli from 2 eval(s) +Running with 5 workers + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ async_upload_with_context_manager [variant: sonnet_skill] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 47,643 + Turns 3 + Tool calls 3 + Wall time 27.4s + Errors 0 + Skills used 1 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The agent produced a well-structured, idiomatic async Python file that correctly uses the azure-monitor-ingestion async API, keeps secrets in environment variables, and provides explicit exception handling with no bare excepts. The only minor gap is that `ValidationError` is not explicitly handled and the outer except is a broad `Exception`, but these are minor issues that don't significantly detract from the quality of the solution. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ sync_basic_upload [variant: sonnet_baseline] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 28,687 + Turns 2 + Tool calls 2 + Wall time 23.2s + Errors 0 + Skills used 0 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The generated code is well-structured, complete, and runnable. It correctly uses the Azure Monitor Ingestion SDK with DefaultAzureCredential, reads all configuration from environment variables, and includes layered explicit exception handling covering all required types. The only notable weakness is importing from the private azure.monitor.ingestion._exceptions module rather than the public SDK surface, which is a minor but real best-practice concern. Apart from that one issue, the code is idiomatic Python and satisfies all the stated requirements. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ async_upload_with_context_manager [variant: sonnet_baseline] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 29,159 + Turns 2 + Tool calls 2 + Wall time 30.6s + Errors 0 + Skills used 0 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The output is a high-quality, idiomatic async Python example using the correct Azure SDK classes and patterns. It uses `async with`, `await upload`, proper context managers, meaningful exception handling with no bare `except`, and clean Python conventions. The two small shortcomings ΓÇö missing `ValidationError` handling and hardcoded endpoint/rule-id placeholders instead of environment variable reads ΓÇö prevent a perfect score but do not significantly diminish the practical utility or correctness of the example. + + All graders passed. + +ΓöüΓöüΓöü azure-monitor-ingestion-py-eval ┬╖ sync_basic_upload [variant: sonnet_skill] [claude-sonnet-4.6] ΓöüΓöüΓöü + Metrics + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Tokens 47,828 + Turns 3 + Tool calls 3 + Wall time 29.3s + Errors 0 + Skills used 1 + Model claude-sonnet-4.6 + + Graders (3/3) + ΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇΓöÇ + Γ£ö Python syntax check Command 'node .vally/tools/check-python-syntax.mjs' passed all checks + Γ£ö Python idiomatic static checks Command 'node .vally/tools/check-python-idiomatic.mjs' passed all checks + Γ£ö Idiomatic Python quality The generated code is a high-quality, idiomatic Python example that correctly uses the Azure Monitor Ingestion SDK. It covers all required elements: env-var-based configuration, DefaultAzureCredential, the upload() API with an on_error callback, a context manager for client lifecycle, and explicit exception handling without bare excepts or wildcard imports. The only minor gap is that ValidationError and AzureError are not listed as named exception types (only caught by the broad fallback), which slightly misses the task's explicit requirement but does not compromise correctness or safety. + + All graders passed. + + Γ£ö azure-monitor-ingestion-py-eval [claude-sonnet-4.6] score: 100.0% (threshold: 75.0%) + + Γ£ö azure-monitor-ingestion-py-eval [claude-sonnet-4.6] score: 91.7% (threshold: 75.0%) + +Saved artifacts +Markdown ΓåÆ Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-40-22-287Z\report.md + +Γ£ö sonnet_skill: 1/1 evals passed +Γ£ö sonnet_baseline: 1/1 evals passed + +Run ID: 3fde020f-faf4-4f9c-9c34-e50919414d41 +Output: Q:\src\skills\tests\scenarios\azure-monitor-ingestion-py\vally\vally-experiment-results\2026-06-30T23-40-22-287Z diff --git a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/scenarios.yaml b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/scenarios.yaml index a61c3d07..26dffd41 100644 --- a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/scenarios.yaml +++ b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/eval.yaml b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/eval.yaml new file mode 100644 index 00000000..0c7611db --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-monitor-opentelemetry-exporter-py-eval +description: Vally evaluation suite for azure-monitor-opentelemetry-exporter-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: trace_exporter_basic + prompt: |- + Create a minimal trace exporter example using AzureMonitorTraceExporter. + Use BatchSpanProcessor, TracerProvider, and emit a span. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: metric_exporter_basic + prompt: |- + Show how to configure AzureMonitorMetricExporter with a + PeriodicExportingMetricReader and record a counter metric. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-opentelemetry-exporter-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..693e4f50 --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-monitor-opentelemetry-exporter-py-eval +description: Vally evaluation suite for azure-monitor-opentelemetry-exporter-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: trace_exporter_basic + prompt: |- + Create a minimal trace exporter example using AzureMonitorTraceExporter. + Use BatchSpanProcessor, TracerProvider, and emit a span. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: metric_exporter_basic + prompt: |- + Show how to configure AzureMonitorMetricExporter with a + PeriodicExportingMetricReader and record a counter metric. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-opentelemetry-exporter-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..9b591da9 --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-exporter-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-monitor-opentelemetry-exporter-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-exporter-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-monitor-opentelemetry-py/scenarios.yaml b/tests/scenarios/azure-monitor-opentelemetry-py/scenarios.yaml index 9e8abaf1..03c57c48 100644 --- a/tests/scenarios/azure-monitor-opentelemetry-py/scenarios.yaml +++ b/tests/scenarios/azure-monitor-opentelemetry-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-monitor-opentelemetry-py/vally/eval.yaml b/tests/scenarios/azure-monitor-opentelemetry-py/vally/eval.yaml new file mode 100644 index 00000000..fee8f0bb --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-monitor-opentelemetry-py-eval +description: Vally evaluation suite for azure-monitor-opentelemetry-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: one_line_setup + prompt: |- + Provide the minimal one-line setup for Azure Monitor OpenTelemetry in Python. + Use configure_azure_monitor and assume the connection string is in environment variables. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: explicit_connection_string + prompt: |- + Show configure_azure_monitor with an explicit connection string literal. + Include InstrumentationKey and IngestionEndpoint. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-opentelemetry-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..c819bccb --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-monitor-opentelemetry-py-eval +description: Vally evaluation suite for azure-monitor-opentelemetry-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: one_line_setup + prompt: |- + Provide the minimal one-line setup for Azure Monitor OpenTelemetry in Python. + Use configure_azure_monitor and assume the connection string is in environment variables. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: explicit_connection_string + prompt: |- + Show configure_azure_monitor with an explicit connection string literal. + Include InstrumentationKey and IngestionEndpoint. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-opentelemetry-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..dbf12bf2 --- /dev/null +++ b/tests/scenarios/azure-monitor-opentelemetry-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-monitor-opentelemetry-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-opentelemetry-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-monitor-query-py/scenarios.yaml b/tests/scenarios/azure-monitor-query-py/scenarios.yaml index 0ecca5dc..c0c43391 100644 --- a/tests/scenarios/azure-monitor-query-py/scenarios.yaml +++ b/tests/scenarios/azure-monitor-query-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-monitor-query-py/vally/eval.yaml b/tests/scenarios/azure-monitor-query-py/vally/eval.yaml new file mode 100644 index 00000000..73b941eb --- /dev/null +++ b/tests/scenarios/azure-monitor-query-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-monitor-query-py-eval +description: Vally evaluation suite for azure-monitor-query-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: logs_basic_query + prompt: |- + Provide a basic LogsQueryClient example that queries a Log Analytics workspace. + Use DefaultAzureCredential and a relative timespan. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: logs_timespan_tuple + prompt: |- + Show a LogsQueryClient example that uses a time range tuple with timezone-aware datetimes. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-query-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-query-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d7da6e68 --- /dev/null +++ b/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,137 @@ +name: azure-monitor-query-py-eval +description: Vally evaluation suite for azure-monitor-query-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: logs_basic_query + prompt: |- + Provide a basic LogsQueryClient example that queries a Log Analytics workspace. + Use DefaultAzureCredential and a relative timespan. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: logs_timespan_tuple + prompt: |- + Show a LogsQueryClient example that uses a time range tuple with timezone-aware datetimes. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-monitor-query-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-query-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..e2fefce1 --- /dev/null +++ b/tests/scenarios/azure-monitor-query-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-monitor-query-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-monitor-query-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-search-documents-py/scenarios.yaml b/tests/scenarios/azure-search-documents-py/scenarios.yaml index 1e9bd997..7cd61531 100644 --- a/tests/scenarios/azure-search-documents-py/scenarios.yaml +++ b/tests/scenarios/azure-search-documents-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-search-documents-py/vally/eval.yaml b/tests/scenarios/azure-search-documents-py/vally/eval.yaml new file mode 100644 index 00000000..48f43b36 --- /dev/null +++ b/tests/scenarios/azure-search-documents-py/vally/eval.yaml @@ -0,0 +1,139 @@ +name: azure-search-documents-py-eval +description: Vally evaluation suite for azure-search-documents-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_setup + prompt: |- + Create a basic Azure AI Search client using SearchClient with proper + authentication. Include connecting to an existing search index and + executing a simple search query. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: vector_search_basic + prompt: |- + Create a vector search example using Azure AI Search. + Generate embeddings for a query and search using VectorizedQuery. + Include proper vector configuration and search parameters. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-search-documents-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-search-documents-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..10154d41 --- /dev/null +++ b/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,140 @@ +name: azure-search-documents-py-eval +description: Vally evaluation suite for azure-search-documents-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_client_setup + prompt: |- + Create a basic Azure AI Search client using SearchClient with proper + authentication. Include connecting to an existing search index and + executing a simple search query. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: vector_search_basic + prompt: |- + Create a vector search example using Azure AI Search. + Generate embeddings for a query and search using VectorizedQuery. + Include proper vector configuration and search parameters. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and query/ingestion operations in try/except blocks + * Handle: ServiceError, HttpResponseError, ValidationError, AzureError + * Example: try: results = client.query_workspace(query) except HttpResponseError as e: print(f'Query failed: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-search-documents-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-search-documents-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..6297c471 --- /dev/null +++ b/tests/scenarios/azure-search-documents-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-search-documents-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-search-documents-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-servicebus-py/scenarios.yaml b/tests/scenarios/azure-servicebus-py/scenarios.yaml index 69c7b216..6cdff8c6 100644 --- a/tests/scenarios/azure-servicebus-py/scenarios.yaml +++ b/tests/scenarios/azure-servicebus-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-servicebus-py/vally/eval.yaml b/tests/scenarios/azure-servicebus-py/vally/eval.yaml new file mode 100644 index 00000000..ea81647e --- /dev/null +++ b/tests/scenarios/azure-servicebus-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-servicebus-py-eval +description: Vally evaluation suite for azure-servicebus-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: sync_queue_send_receive + prompt: |- + Create a sync example that sends a message to a queue and receives it. + Use DefaultAzureCredential and context managers. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: async_queue_send_receive + prompt: |- + Create an async example that sends and receives messages from a queue. + Use async clients and await all operations. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-servicebus-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-servicebus-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..6aac7c92 --- /dev/null +++ b/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-servicebus-py-eval +description: Vally evaluation suite for azure-servicebus-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: sync_queue_send_receive + prompt: |- + Create a sync example that sends a message to a queue and receives it. + Use DefaultAzureCredential and context managers. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: async_queue_send_receive + prompt: |- + Create an async example that sends and receives messages from a queue. + Use async clients and await all operations. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap client initialization and send/receive operations in try/except blocks + * Handle: ServiceBusError, EventHubError, OperationTimeoutError, AzureError + * Example: try: sender.send_batch(batch) except ServiceBusError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-servicebus-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-servicebus-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..6c955f20 --- /dev/null +++ b/tests/scenarios/azure-servicebus-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-servicebus-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-servicebus-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-servicebus-rust/acceptance-criteria.md b/tests/scenarios/azure-servicebus-rust/acceptance-criteria.md deleted file mode 100644 index 58aa51e9..00000000 --- a/tests/scenarios/azure-servicebus-rust/acceptance-criteria.md +++ /dev/null @@ -1,124 +0,0 @@ -# Azure Service Bus SDK for Rust Acceptance Criteria - -**Crate**: `azure_messaging_servicebus` -**Repository**: -**Purpose**: Skill testing acceptance criteria for validating generated Rust code correctness - ---- - -## 0. Dependency Management Gate (Required) - -### 0.1 ✅ CORRECT: Use cargo commands for dependency changes - -```sh -cargo add azure_messaging_servicebus azure_identity tokio -cargo add azure_core -cargo remove azure_core -``` - -### 0.2 ✅ CORRECT: Add `azure_core` only for direct `azure_core` imports - -```rust -use azure_core::error::ErrorKind; -use azure_messaging_servicebus::ServiceBusClient; -// Direct azure_core import is used, so `azure_core` should be a direct dependency. -``` - -### 0.3 ❌ INCORRECT: Manual Cargo.toml dependency edits in generated guidance - -```toml -# WRONG in generated guidance - use `cargo add` / `cargo remove` commands instead -[dependencies] -azure_core = "*" -``` - -### 0.4 ❌ INCORRECT: Requiring `azure_core` when no direct `azure_core` imports exist - -```rust -use azure_messaging_servicebus::ServiceBusClient; -// No direct azure_core import here, so forcing direct azure_core dependency is unnecessary. -``` - ---- - -## 1. Correct Import Patterns - -### 1.1 ✅ CORRECT: Client and message imports - -```rust -use azure_messaging_servicebus::{ServiceBusClient, Message}; -use azure_identity::DeveloperToolsCredential; -``` - ---- - -## 2. Client Creation - -### 2.1 ✅ CORRECT: ServiceBusClient builder and open - -```rust -let credential = DeveloperToolsCredential::new(None)?; -let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; -``` - -### 2.2 ❌ INCORRECT: Direct constructor call instead of builder pattern - -```rust -// WRONG - use ServiceBusClient::builder().open(...) -let client = ServiceBusClient::new("namespace", credential)?; -``` - ---- - -## 3. Queue and Topic Operations - -### 3.1 ✅ CORRECT: Send message to queue - -```rust -let sender = client.create_sender("my_queue", None).await?; -let message = Message::from("Hello, Service Bus!"); -sender.send_message(message, None).await?; -``` - -### 3.2 ✅ CORRECT: Receive and complete queue messages - -```rust -let receiver = client.create_receiver("my_queue", None).await?; -let messages = receiver.receive_messages(5, None).await?; -for message in messages { - println!("Received: {}", message.body_as_string()?); - receiver.complete_message(&message, None).await?; -} -``` - -### 3.3 ✅ CORRECT: Topic/subscription receive - -```rust -let receiver = client - .create_receiver_for_subscription("my_topic", "my_subscription", None) - .await?; - -let messages = receiver.receive_messages(5, None).await?; -for message in messages { - receiver.complete_message(&message, None).await?; -} -``` - -### 3.4 ❌ INCORRECT: Forgetting message settlement - -```rust -// WRONG - leaving messages unsettled causes redelivery -for message in receiver.receive_messages(5, None).await? { - println!("{}", message.body_as_string()?); -} -``` - ---- - -## 4. Package Maturity Warning - -### 4.1 ✅ CORRECT: Include pre-production warning - -Generated guidance should mention this crate is in early development and should not be used in production without caution. diff --git a/tests/scenarios/azure-servicebus-rust/scenarios.yaml b/tests/scenarios/azure-servicebus-rust/scenarios.yaml deleted file mode 100644 index 2fc35c8e..00000000 --- a/tests/scenarios/azure-servicebus-rust/scenarios.yaml +++ /dev/null @@ -1,125 +0,0 @@ -# yaml-language-server: $schema=../skill-scenarios.schema.json -# Test scenarios for azure-servicebus-rust skill evaluation -# Each scenario tests a specific usage pattern against acceptance criteria - -config: - model: claude-sonnet-4.6 - max_tokens: 2000 - temperature: 0.3 - -scenarios: - - name: dependency_gate_cargo_add - prompt: | - For a Rust Azure Service Bus example, explain dependency management rules. - expected_patterns: - - "cargo add" - - "Cargo.toml" - - "azure_core" - forbidden_patterns: - - "edit Cargo.toml manually" - tags: - - dependency - - guardrail - mock_response: | - Use cargo commands for dependency changes, for example: - cargo add azure_messaging_servicebus azure_identity tokio - Add azure_core only when importing azure_core types directly. - Do not manually edit Cargo.toml in generated guidance. - - - name: create_servicebus_client - prompt: | - Create a ServiceBusClient in Rust using DeveloperToolsCredential. - expected_patterns: - - "ServiceBusClient" - - "builder" - - "open" - - "DeveloperToolsCredential" - - "servicebus.windows.net" - forbidden_patterns: - - "ServiceBusClient::new" - tags: - - basic - - authentication - mock_response: | - use azure_identity::DeveloperToolsCredential; - use azure_messaging_servicebus::ServiceBusClient; - - let credential = DeveloperToolsCredential::new(None)?; - let client = ServiceBusClient::builder() - .open("your_namespace.servicebus.windows.net", credential.clone()) - .await?; - - - name: send_queue_message - prompt: | - Send a message to a Service Bus queue in Rust. - expected_patterns: - - "create_sender" - - "Message::from" - - "send_message" - forbidden_patterns: - - "receive_messages" - tags: - - send - - queue - mock_response: | - use azure_messaging_servicebus::Message; - - let sender = client.create_sender("my_queue", None).await?; - let message = Message::from("Hello, Service Bus!"); - sender.send_message(message, None).await?; - - - name: receive_and_complete_queue_messages - prompt: | - Receive queue messages and settle them correctly with complete_message. - expected_patterns: - - "create_receiver" - - "receive_messages" - - "complete_message" - tags: - - receive - - settlement - - queue - mock_response: | - let receiver = client.create_receiver("my_queue", None).await?; - let messages = receiver.receive_messages(5, None).await?; - for message in messages { - println!("Received: {}", message.body_as_string()?); - receiver.complete_message(&message, None).await?; - } - - - name: receive_subscription_messages - prompt: | - Receive messages from a topic subscription in Rust Service Bus. - expected_patterns: - - "create_receiver_for_subscription" - - "receive_messages" - - "complete_message" - forbidden_patterns: - - "create_sender" - tags: - - topic - - subscription - - receive - mock_response: | - let receiver = client - .create_receiver_for_subscription("my_topic", "my_subscription", None) - .await?; - - let messages = receiver.receive_messages(5, None).await?; - for message in messages { - receiver.complete_message(&message, None).await?; - } - - - name: package_maturity_warning - prompt: | - What production caution should be included when using azure_messaging_servicebus in Rust? - expected_patterns: - - "early development" - - "should not be used in production" - forbidden_patterns: - - "production-ready" - tags: - - warning - - maturity - mock_response: | - This crate is in early development and should not be used in production without caution, because APIs may change. diff --git a/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml deleted file mode 100644 index 98656955..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,163 +0,0 @@ -name: azure-servicebus-rust-eval -description: Evaluation suite for azure-messaging-servicebus-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-servicebus-messaging - prompt: | - Write a Rust program named rust-servicebus-ops that demonstrates Azure Service Bus - messaging patterns including queues, topics, and subscriptions: - - Queue operations: - 1. Create a ServiceBusClient with namespace credentials - 2. Create a sender for a queue named "orders" - 3. Send messages to the queue via sender.send_message() - 4. Create a receiver for the queue - 5. Receive messages (up to 5) via receiver.receive_messages() - 6. Complete messages after processing via receiver.complete_message() - 7. Handle cases where messages must be abandoned for retry - - Topic and subscription operations: - 1. Create a sender for a topic named "notifications" - 2. Send messages to the topic - 3. Create a receiver for a subscription "alerts" on the topic - 4. Receive messages from the subscription - 5. Complete messages after processing - - Demonstrate: - - ServiceBusClient::builder().open(namespace, credential).await pattern - - Message creation via Message::from() - - Sender/receiver creation for both queues and topics - - Message body access via body_as_string() - - Message settlement: complete_message() for success, abandon for retry - - TokenCredential authentication (DeveloperToolsCredential/ManagedIdentityCredential) - - Async/await with Tokio runtime - - Proper Result-based error propagation - - The program should compile and run successfully with proper async message handling, - demonstrating the complete Service Bus workflow for enterprise messaging. - - rubric: - - "Uses official azure_messaging_servicebus SDK with proper client creation." - - "Implements ServiceBusClient with TokenCredential-based authentication." - - "Creates senders and receivers for both queues and topics." - - "Sends messages to queues and topics via sender.send_message()." - - "Receives messages with receiver.receive_messages(count, options)." - - "Completes messages after processing with receiver.complete_message()." - - "Handles message bodies with body_as_string() or equivalent extraction." - - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - - "Async/await with Tokio runtime and proper error handling." - - "Code compiles without warnings and follows Rust idioms." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-servicebus-rust - graders: - - type: prompt - name: "Service Bus messaging completeness" - config: - model: gpt-5.3-codex - prompt: | - - ServiceBusClient initialization with builder and open() - - Queue sender/receiver creation - - Topic sender and subscription receiver creation - - Message sending via send_message(message, None).await - - Message receiving via receive_messages(count, None).await - - Message body extraction via body_as_string() - - Message settlement with complete_message() - - Abandon message pattern for retries - - TokenCredential authentication pattern - - Async/await with Tokio #[tokio::main] - scoring: scale_1_5 - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: ServiceBusClient Builder Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ServiceBusClient created via ServiceBusClient::builder() - .open(namespace, credential).await? - Sender created via client.create_sender(queue_or_topic_name, None).await? - Receiver created via client.create_receiver(queue_name, None).await? - Topic subscription receiver via create_receiver_for_subscription(topic, subscription, None).await? - - name: Message Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Messages created via Message::from(text) or Message::from(bytes) - Send via sender.send_message(message, None).await? - Receive via receiver.receive_messages(count, None).await? returning Vec - Extract body via message.body_as_string()? or equivalent - - name: Message Settlement - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - After processing, call receiver.complete_message(&message, None).await? - For retriable errors, use receiver.abandon_message(&message, None).await? - No message is processed without explicit settlement - Proper Result-based error handling for settlement operations - - name: Queue and Topic Patterns - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Queues demonstrated via create_sender/create_receiver on queue names - Topics demonstrated via create_sender on topic names - Subscriptions accessed via create_receiver_for_subscription(topic_name, subscription_name, None).await? - Both patterns use same Message type and settlement semantics - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Uses #[tokio::main] attribute on main function - All I/O operations properly awaited - No blocking operations in async context - Result error handling throughout diff --git a/tests/scenarios/azure-servicebus-rust/vally/eval.yaml b/tests/scenarios/azure-servicebus-rust/vally/eval.yaml deleted file mode 100644 index 93e4e130..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/eval.yaml +++ /dev/null @@ -1,192 +0,0 @@ -name: azure-servicebus-rust-eval -description: Evaluation suite for azure-messaging-servicebus-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 0.75 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-servicebus-messaging - prompt: | - Write a Rust program named rust-servicebus-ops that demonstrates Azure Service Bus - messaging patterns including queues, topics, and subscriptions: - - Queue operations: - 1. Create a ServiceBusClient with namespace credentials - 2. Create a sender for a queue named "orders" - 3. Send messages to the queue via sender.send_message() - 4. Create a receiver for the queue - 5. Receive messages (up to 5) via receiver.receive_messages() - 6. Complete messages after processing via receiver.complete_message() - 7. Handle cases where messages must be abandoned for retry - - Topic and subscription operations: - 1. Create a sender for a topic named "notifications" - 2. Send messages to the topic - 3. Create a receiver for a subscription "alerts" on the topic - 4. Receive messages from the subscription - 5. Complete messages after processing - - Demonstrate: - - ServiceBusClient::builder().open(namespace, credential).await pattern - - Message creation via Message::from() - - Sender/receiver creation for both queues and topics - - Message body access via body_as_string() - - Message settlement: complete_message() for success, abandon for retry - - TokenCredential authentication (DeveloperToolsCredential/ManagedIdentityCredential) - - Async/await with Tokio runtime - - Proper Result-based error propagation - - The program should compile and run successfully with proper async message handling, - demonstrating the complete Service Bus workflow for enterprise messaging. - - rubric: - - "Uses official azure_messaging_servicebus SDK with proper client creation." - - "Implements ServiceBusClient with TokenCredential-based authentication." - - "Creates senders and receivers for both queues and topics." - - "Sends messages to queues and topics via sender.send_message()." - - "Receives messages with receiver.receive_messages(count, options)." - - "Completes messages after processing with receiver.complete_message()." - - "Handles message bodies with body_as_string() or equivalent extraction." - - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - - "Async/await with Tokio runtime and proper error handling." - - "Code compiles without warnings and follows Rust idioms." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-servicebus-rust - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-servicebus-rust - graders: - - type: prompt - name: "Service Bus messaging completeness" - config: - model: gpt-5.3-codex - prompt: | - - ServiceBusClient initialization with builder and open() - - Queue sender/receiver creation - - Topic sender and subscription receiver creation - - Message sending via send_message(message, None).await - - Message receiving via receive_messages(count, None).await - - Message body extraction via body_as_string() - - Message settlement with complete_message() - - Abandon message pattern for retries - - TokenCredential authentication pattern - - Async/await with Tokio #[tokio::main] - scoring: scale_1_5 - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" - - name: Async-First with Tokio Runtime - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" - - name: No Hardcoded Secrets - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: ServiceBusClient Builder Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ServiceBusClient created via ServiceBusClient::builder() - .open(namespace, credential).await? - Sender created via client.create_sender(queue_or_topic_name, None).await? - Receiver created via client.create_receiver(queue_name, None).await? - Topic subscription receiver via create_receiver_for_subscription(topic, subscription, None).await? - - name: Message Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Messages created via Message::from(text) or Message::from(bytes) - Send via sender.send_message(message, None).await? - Receive via receiver.receive_messages(count, None).await? returning Vec - Extract body via message.body_as_string()? or equivalent - - name: Message Settlement - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - After processing, call receiver.complete_message(&message, None).await? - For retriable errors, use receiver.abandon_message(&message, None).await? - No message is processed without explicit settlement - Proper Result-based error handling for settlement operations - - name: Queue and Topic Patterns - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Queues demonstrated via create_sender/create_receiver on queue names - Topics demonstrated via create_sender on topic names - Subscriptions accessed via create_receiver_for_subscription(topic_name, subscription_name, None).await? - Both patterns use same Message type and settlement semantics diff --git a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml deleted file mode 100644 index 6d37af5b..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/skill_effectiveness_eval.yaml +++ /dev/null @@ -1,199 +0,0 @@ -name: azure-servicebus-rust-eval -description: Evaluation suite for azure-messaging-servicebus-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 0.75 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-servicebus-messaging - prompt: | - Write a Rust program named rust-servicebus-ops that demonstrates Azure Service Bus - messaging patterns including queues, topics, and subscriptions: - - Queue operations: - 1. Create a ServiceBusClient with namespace credentials - 2. Create a sender for a queue named "orders" - 3. Send messages to the queue via sender.send_message() - 4. Create a receiver for the queue - 5. Receive messages (up to 5) via receiver.receive_messages() - 6. Complete messages after processing via receiver.complete_message() - 7. Handle cases where messages must be abandoned for retry - - Topic and subscription operations: - 1. Create a sender for a topic named "notifications" - 2. Send messages to the topic - 3. Create a receiver for a subscription "alerts" on the topic - 4. Receive messages from the subscription - 5. Complete messages after processing - - Demonstrate: - - ServiceBusClient::builder().open(namespace, credential).await pattern - - Message creation via Message::from() - - Sender/receiver creation for both queues and topics - - Message body access via body_as_string() - - Message settlement: complete_message() for success, abandon for retry - - TokenCredential authentication (DeveloperToolsCredential/ManagedIdentityCredential) - - Async/await with Tokio runtime - - Proper Result-based error propagation - - The program should compile and run successfully with proper async message handling, - demonstrating the complete Service Bus workflow for enterprise messaging. - - rubric: - - "Uses official azure_messaging_servicebus SDK with proper client creation." - - "Implements ServiceBusClient with TokenCredential-based authentication." - - "Creates senders and receivers for both queues and topics." - - "Sends messages to queues and topics via sender.send_message()." - - "Receives messages with receiver.receive_messages(count, options)." - - "Completes messages after processing with receiver.complete_message()." - - "Handles message bodies with body_as_string() or equivalent extraction." - - "Demonstrates both queue (point-to-point) and topic (pub-sub) patterns." - - "Async/await with Tokio runtime and proper error handling." - - "Code compiles without warnings and follows Rust idioms." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/src/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - - src: ../../_shared/vally/tools/check-azure-crates.rs - dest: .vally/tools/check-azure-crates.rs - - src: ../../_shared/vally/tools/check-token-credential.rs - dest: .vally/tools/check-token-credential.rs - - src: ../../_shared/vally/tools/check-async-runtime.rs - dest: .vally/tools/check-async-runtime.rs - - src: ../../_shared/vally/tools/check-no-secrets.rs - dest: .vally/tools/check-no-secrets.rs - constraints: - expect_skills: - - azure-servicebus-rust - graders: - - type: prompt - name: "Service Bus messaging completeness" - config: - model: gpt-5.3-codex - prompt: | - - ServiceBusClient initialization with builder and open() - - Queue sender/receiver creation - - Topic sender and subscription receiver creation - - Message sending via send_message(message, None).await - - Message receiving via receive_messages(count, None).await - - Message body extraction via body_as_string() - - Message settlement with complete_message() - - Abandon message pattern for retries - - TokenCredential authentication pattern - - Async/await with Tokio #[tokio::main] - scoring: scale_1_5 - - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" - - name: Async-First with Tokio Runtime - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" - - name: No Hardcoded Secrets - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" - - - type: run-command - name: "Cargo Check" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - type: run-command - name: "Cargo Build" - config: - command: cargo - args: - - "build" - timeout: 120s - - - name: ServiceBusClient Builder Pattern - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - ServiceBusClient created via ServiceBusClient::builder() - .open(namespace, credential).await? - Sender created via client.create_sender(queue_or_topic_name, None).await? - Receiver created via client.create_receiver(queue_name, None).await? - Topic subscription receiver via create_receiver_for_subscription(topic, subscription, None).await? - - name: Message Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Messages created via Message::from(text) or Message::from(bytes) - Send via sender.send_message(message, None).await? - Receive via receiver.receive_messages(count, None).await? returning Vec - Extract body via message.body_as_string()? or equivalent - - name: Message Settlement - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - After processing, call receiver.complete_message(&message, None).await? - For retriable errors, use receiver.abandon_message(&message, None).await? - No message is processed without explicit settlement - Proper Result-based error handling for settlement operations - - name: Queue and Topic Patterns - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Queues demonstrated via create_sender/create_receiver on queue names - Topics demonstrated via create_sender on topic names - Subscriptions accessed via create_receiver_for_subscription(topic_name, subscription_name, None).await? - Both patterns use same Message type and settlement semantics diff --git a/tests/scenarios/azure-servicebus-rust/vally/workspace/cargo.toml b/tests/scenarios/azure-servicebus-rust/vally/workspace/cargo.toml deleted file mode 100644 index daa8c449..00000000 --- a/tests/scenarios/azure-servicebus-rust/vally/workspace/cargo.toml +++ /dev/null @@ -1,2 +0,0 @@ -[workspace] -resolver = "3" diff --git a/tests/scenarios/azure-speech-to-text-rest-py/scenarios.yaml b/tests/scenarios/azure-speech-to-text-rest-py/scenarios.yaml index 397ff312..63f26827 100644 --- a/tests/scenarios/azure-speech-to-text-rest-py/scenarios.yaml +++ b/tests/scenarios/azure-speech-to-text-rest-py/scenarios.yaml @@ -1,6 +1,6 @@ config: max_tokens: 2000 - model: gpt-4 + model: gpt-5.5 temperature: 0.3 scenarios: - expected_patterns: diff --git a/tests/scenarios/azure-speech-to-text-rest-py/vally/eval.yaml b/tests/scenarios/azure-speech-to-text-rest-py/vally/eval.yaml new file mode 100644 index 00000000..b8a76cb4 --- /dev/null +++ b/tests/scenarios/azure-speech-to-text-rest-py/vally/eval.yaml @@ -0,0 +1,86 @@ +name: azure-speech-to-text-rest-py-eval +description: Vally evaluation suite for azure-speech-to-text-rest-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-speech-to-text-rest-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-speech-to-text-rest-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..b2484675 --- /dev/null +++ b/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,87 @@ +name: azure-speech-to-text-rest-py-eval +description: Vally evaluation suite for azure-speech-to-text-rest-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: core_workflow + prompt: |- + Write a Python example that demonstrates the core workflow for the azure-speech-to-text-rest-py skill using the official SDK patterns, environment variables, and explicit error handling. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all API operations in try/except blocks + * Handle: AzureError, HttpResponseError, ServiceError, and generic Exception types + * Example: try: result = client.analyze(...) except HttpResponseError as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-speech-to-text-rest-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..bca2d438 --- /dev/null +++ b/tests/scenarios/azure-speech-to-text-rest-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-speech-to-text-rest-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-speech-to-text-rest-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-blob-py/scenarios.yaml b/tests/scenarios/azure-storage-blob-py/scenarios.yaml index 0fdd1a07..36f505bd 100644 --- a/tests/scenarios/azure-storage-blob-py/scenarios.yaml +++ b/tests/scenarios/azure-storage-blob-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-storage-blob-py/vally/eval.yaml b/tests/scenarios/azure-storage-blob-py/vally/eval.yaml new file mode 100644 index 00000000..46e7f6e9 --- /dev/null +++ b/tests/scenarios/azure-storage-blob-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-storage-blob-py-eval +description: Vally evaluation suite for azure-storage-blob-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_blob_service_client + prompt: |- + Create a basic Azure Blob Storage example that authenticates with + DefaultAzureCredential and initializes a BlobServiceClient. Then + create a container client and create the container. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: upload_blob_with_blob_client + prompt: |- + Upload a local file to Azure Blob Storage using BlobClient. + Use overwrite=True when uploading. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-blob-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-blob-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..94ee8d0f --- /dev/null +++ b/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-storage-blob-py-eval +description: Vally evaluation suite for azure-storage-blob-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_blob_service_client + prompt: |- + Create a basic Azure Blob Storage example that authenticates with + DefaultAzureCredential and initializes a BlobServiceClient. Then + create a container client and create the container. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: upload_blob_with_blob_client + prompt: |- + Upload a local file to Azure Blob Storage using BlobClient. + Use overwrite=True when uploading. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-blob-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-blob-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..89f97928 --- /dev/null +++ b/tests/scenarios/azure-storage-blob-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-storage-blob-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-blob-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml index b0248616..c1da674a 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval-keyvault-encryption.yaml @@ -7,7 +7,7 @@ scoring: threshold: 1.0 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 stimuli: - name: rust-storage-keyvault-encryption-library @@ -39,7 +39,7 @@ stimuli: - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - "Code compiles and passes cargo check for the generated library project." - "Avoids hardcoded secrets in source code or committed configuration." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -100,44 +100,36 @@ stimuli: - Reasonable error handling and no embedded secrets scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true timeout: 2m - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true timeout: 2m - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true timeout: 2m - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true timeout: 2m - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml deleted file mode 100644 index 0045654b..00000000 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild-experiment.yaml +++ /dev/null @@ -1,268 +0,0 @@ -name: azure-storage-blob-rust-nobuild-eval -description: Evaluation suite for azure-storage-blob-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-crud-blobs-nobuild - prompt: | - Write a Rust program named rust-blob-nobuild that performs the following blob storage operations: - 1. Create a blob container called "my-container" if it does not already exist - 2. Upload a local file "example.txt" as a block blob named "uploads/example.txt" - 3. List all blobs in the container and print their names and sizes - 4. Download the blob back to a local file "downloaded-example.txt" - 5. Delete the blob and then delete the container - - Include proper error handling for cases like container already existing - or blob not found. - rubric: - - "Implements the full blob CRUD workflow end-to-end: create container, upload, list, download, and delete blob/container." - - "Uses official Azure Rust SDK crate patterns and a TokenCredential-based authentication flow." - - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - - "Code compiles and passes cargo check for the rust-blob project." - - "Uses correct Azure client construction, response handling, and pager usage where applicable." - - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/build.rs" - - "**/src/**/*.rs" - - "**/examples/**/*.rs" - - "**/tests/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - - "**/*.d" - - "**/*.o" - - "**/*.obj" - - "**/*.pdb" - - "**/*.exe" - - "**/*.dll" - - "**/*.so" - - "**/*.dylib" - - "**/*.a" - - "**/*.lib" - - "**/*.wasm" - environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-blob-rust - files: - - src: ../../_shared/vally/tools/find-cargo-root.js - dest: .vally/tools/find-cargo-root.js - - src: ../../_shared/vally/tools/check-azure-crates.rs - dest: .vally/tools/check-azure-crates.rs - - src: ../../_shared/vally/tools/check-token-credential.rs - dest: .vally/tools/check-token-credential.rs - - src: ../../_shared/vally/tools/check-async-runtime.rs - dest: .vally/tools/check-async-runtime.rs - - src: ../../_shared/vally/tools/check-no-secrets.rs - dest: .vally/tools/check-no-secrets.rs - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-storage-blob-rust - graders: - - type: prompt - name: "Prompt correctness and completeness of blob storage operations" - config: - model: gpt-5.3-codex - prompt: | - - The code should demonstrate an azure blob storage client with a storage account url and credential - - Creating a container for container creation - - Uploading from a file path - - Enumerate blobs in a container - - Download from a blob to a local file - - Deleting blob files and containers for cleanup - - Error handling - scoring: scale_1_5 - - - type: run-command - name: "Build Application" - config: - command: cargo - args: - - "build" - stderr_contains: "Finished" - timeout: 120s - - type: run-command - name: "Check Application" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 0.5 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" - - name: Async-First with Tokio Runtime - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" - - name: Idiomatic Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Error handling uses Result and the ? operator. Calls to - .unwrap() are avoided or used only where failure - is impossible. Calls to expect() with a reasonable error message are allowed - only if termination of the application is acceptable. Azure service errors are - matched via azure_core::Error with ErrorKind (e.g., - ErrorKind::HttpResponse { status, error_code, .. } with - StatusCode comparison). The main function returns - Result<(), Box> or azure_core::Result<()>. - - name: Correct Client Construction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Service clients are constructed using the ::new() constructor - with an endpoint string, credential, and Option parameter. - If a ClientOptions parameter is specified, structs use ..Default::default() - for forward compatibility. If all the fields in the ClientOptions struct are provided, the - #[allow(clippy::needless_update)] annotation is used. - - name: Pager and Stream Usage - type: prompt - model: gpt-5.3-codex - weight: 1.0 - prompt: > - List operations return and consume azure_core::Pager using - futures::TryStreamExt (try_next/try_collect). Pagination is - handled correctly by iterating all items. In most if not all - cases, individual items should be iterated over directly rather - than page-by-page, demonstrating the SDK's ability to handle - pagination internally. Page-by-page iteration via into_pages() - is only necessary when there is an explicit requirement for page-level - access. - - name: Response Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Calls to service clients correctly handle Response. When service - methods return a model type, code calls into_model() to deserialize - the response body into the expected model type (not manually parsing - JSON). When raw HTTP details are needed, deconstruct() is used to - access status, headers, and body separately. - - name: Long-Running Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Long-running operations use azure_core::Poller correctly. - If a client calls into an API returning a Poller, code awaits - the Poller directly (via IntoFuture) for the final result. - The only time that a client should iterate over status updates via - futures::TryStreamExt is when there is an explicit requirement for - status-level access. - - name: Struct Initialization with Default - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - If a client initializes a structure for SDK types (client options, method - options, model types) the code uses ..Default::default() to protect - against future field additions. - - name: Ownership and Borrowing - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code uses references (&str, &T) for borrowed data and avoids - unnecessary .clone() calls. Credentials are shared via Arc or - clone(). No unnecessary Box or Rc where Arc is appropriate. - Parameters that are only used for URL construction borrow - rather than own the data. - - name: No Hardcoded Secrets - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" - - name: Rust Cargo Trajectory Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - collect_trajectory_compiler_errors: true - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo build - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Check Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo check - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Clippy Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo clippy - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml deleted file mode 100644 index 803bcd3f..00000000 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval-nobuild.yaml +++ /dev/null @@ -1,268 +0,0 @@ -name: azure-storage-blob-rust-nobuild-eval -description: Evaluation suite for azure-storage-blob-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-crud-blobs-nobuild - prompt: | - Write a Rust program named rust-blob-nobuild that performs the following blob storage operations: - 1. Create a blob container called "my-container" if it does not already exist - 2. Upload a local file "example.txt" as a block blob named "uploads/example.txt" - 3. List all blobs in the container and print their names and sizes - 4. Download the blob back to a local file "downloaded-example.txt" - 5. Delete the blob and then delete the container - - Include proper error handling for cases like container already existing - or blob not found. - rubric: - - "Implements the full blob CRUD workflow end-to-end: create container, upload, list, download, and delete blob/container." - - "Uses official Azure Rust SDK crate patterns and a TokenCredential-based authentication flow." - - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - - "Code compiles and passes cargo check for the rust-blob project." - - "Uses correct Azure client construction, response handling, and pager usage where applicable." - - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/build.rs" - - "**/src/**/*.rs" - - "**/examples/**/*.rs" - - "**/tests/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - - "**/*.d" - - "**/*.o" - - "**/*.obj" - - "**/*.pdb" - - "**/*.exe" - - "**/*.dll" - - "**/*.so" - - "**/*.dylib" - - "**/*.a" - - "**/*.lib" - - "**/*.wasm" - environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-blob-rust - files: - - src: ../../_shared/vally/tools/find-cargo-root.js - dest: .vally/tools/find-cargo-root.js - - src: ../../_shared/vally/tools/check-azure-crates.rs - dest: .vally/tools/check-azure-crates.rs - - src: ../../_shared/vally/tools/check-token-credential.rs - dest: .vally/tools/check-token-credential.rs - - src: ../../_shared/vally/tools/check-async-runtime.rs - dest: .vally/tools/check-async-runtime.rs - - src: ../../_shared/vally/tools/check-no-secrets.rs - dest: .vally/tools/check-no-secrets.rs - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-storage-blob-rust - graders: - - type: prompt - name: "Prompt correctness and completeness of blob storage operations" - config: - model: gpt-5.3-codex - prompt: | - - The code should demonstrate an azure blob storage client with a storage account url and credential - - Creating a container for container creation - - Uploading from a file path - - Enumerate blobs in a container - - Download from a blob to a local file - - Deleting blob files and containers for cleanup - - Error handling - scoring: scale_1_5 - - - type: run-command - name: "Build Application" - config: - command: cargo - args: - - "build" - stderr_contains: "Finished" - timeout: 3m - - type: run-command - name: "Check Application" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 0.5 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" - - name: Async-First with Tokio Runtime - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" - - name: Idiomatic Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Error handling uses Result and the ? operator. Calls to - .unwrap() are avoided or used only where failure - is impossible. Calls to expect() with a reasonable error message are allowed - only if termination of the application is acceptable. Azure service errors are - matched via azure_core::Error with ErrorKind (e.g., - ErrorKind::HttpResponse { status, error_code, .. } with - StatusCode comparison). The main function returns - Result<(), Box> or azure_core::Result<()>. - - name: Correct Client Construction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Service clients are constructed using the ::new() constructor - with an endpoint string, credential, and Option parameter. - If a ClientOptions parameter is specified, structs use ..Default::default() - for forward compatibility. If all the fields in the ClientOptions struct are provided, the - #[allow(clippy::needless_update)] annotation is used. - - name: Pager and Stream Usage - type: prompt - model: gpt-5.3-codex - weight: 1.0 - prompt: > - List operations return and consume azure_core::Pager using - futures::TryStreamExt (try_next/try_collect). Pagination is - handled correctly by iterating all items. In most if not all - cases, individual items should be iterated over directly rather - than page-by-page, demonstrating the SDK's ability to handle - pagination internally. Page-by-page iteration via into_pages() - is only necessary when there is an explicit requirement for page-level - access. - - name: Response Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Calls to service clients correctly handle Response. When service - methods return a model type, code calls into_model() to deserialize - the response body into the expected model type (not manually parsing - JSON). When raw HTTP details are needed, deconstruct() is used to - access status, headers, and body separately. - - name: Long-Running Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Long-running operations use azure_core::Poller correctly. - If a client calls into an API returning a Poller, code awaits - the Poller directly (via IntoFuture) for the final result. - The only time that a client should iterate over status updates via - futures::TryStreamExt is when there is an explicit requirement for - status-level access. - - name: Struct Initialization with Default - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - If a client initializes a structure for SDK types (client options, method - options, model types) the code uses ..Default::default() to protect - against future field additions. - - name: Ownership and Borrowing - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code uses references (&str, &T) for borrowed data and avoids - unnecessary .clone() calls. Credentials are shared via Arc or - clone(). No unnecessary Box or Rc where Arc is appropriate. - Parameters that are only used for URL construction borrow - rather than own the data. - - name: No Hardcoded Secrets - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" - - name: Rust Cargo Trajectory Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - collect_trajectory_compiler_errors: true - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo build - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Check Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo check - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Clippy Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo clippy - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail diff --git a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml index 7064a05d..c9eca445 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-crud-blobs prompt: | @@ -37,7 +38,7 @@ stimuli: - "Uses correct Azure client construction, response handling, and pager usage where applicable." - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,7 +84,6 @@ stimuli: - type: prompt name: "Prompt correctness and completeness of blob storage operations" config: - model: gpt-5.3-codex prompt: | - The code should demonstrate an azure blob storage client with a storage account url and credential - Creating a container for container creation @@ -95,36 +95,29 @@ stimuli: scoring: scale_1_5 - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: Idiomatic Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Error handling uses Result and the ? operator. Calls to .unwrap() are avoided or used only where failure @@ -137,7 +130,6 @@ stimuli: - name: Correct Client Construction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Service clients are constructed using the ::new() constructor with an endpoint string, credential, and Option parameter. @@ -146,7 +138,6 @@ stimuli: #[allow(clippy::needless_update)] annotation is used. - name: Pager and Stream Usage type: prompt - model: gpt-5.3-codex weight: 1.0 prompt: > List operations return and consume azure_core::Pager using @@ -160,7 +151,6 @@ stimuli: - name: Response Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Calls to service clients correctly handle Response. When service methods return a model type, code calls into_model() to deserialize @@ -170,7 +160,6 @@ stimuli: - name: Long-Running Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Long-running operations use azure_core::Poller correctly. If a client calls into an API returning a Poller, code awaits @@ -181,7 +170,6 @@ stimuli: - name: Struct Initialization with Default type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > If a client initializes a structure for SDK types (client options, method options, model types) the code uses ..Default::default() to protect @@ -189,7 +177,6 @@ stimuli: - name: Ownership and Borrowing type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Code uses references (&str, &T) for borrowed data and avoids unnecessary .clone() calls. Credentials are shared via Arc or @@ -197,14 +184,12 @@ stimuli: Parameters that are only used for URL construction borrow rather than own the data. - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true - name: Rust Cargo Trajectory Build Failure type: rust-cargo-build-failure-check weight: 1.0 diff --git a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml index e04f29f1..08818e29 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-crud-blobs prompt: | @@ -37,7 +38,7 @@ stimuli: - "Uses correct Azure client construction, response handling, and pager usage where applicable." - "Avoids hardcoded secrets in source code or configuration committed to the workspace." - "rustfmt compliance is bonus quality signal only and should not block a successful evaluation when all substantive correctness and SDK-usage criteria pass." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -61,8 +62,6 @@ stimuli: - "**/*.lib" - "**/*.wasm" environment: - skills: - - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-blob-rust files: - src: ../../_shared/vally/tools/find-cargo-root.js dest: .vally/tools/find-cargo-root.js @@ -83,7 +82,7 @@ stimuli: - type: prompt name: "Prompt correctness and completeness of blob storage operations" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - The code should demonstrate an azure blob storage client with a storage account url and credential - Creating a container for container creation @@ -100,7 +99,7 @@ stimuli: command: cargo args: - "build" - timeout: 120s + timeout: 3m stderr_contains: "Finished" - type: run-command name: "Check Application" @@ -108,7 +107,7 @@ stimuli: command: cargo args: - "check" - timeout: 90s + timeout: 3m - name: Rust Clippy Lints type: run-command weight: 0.5 @@ -150,7 +149,7 @@ stimuli: - name: Idiomatic Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Error handling uses Result and the ? operator. Calls to .unwrap() are avoided or used only where failure @@ -163,7 +162,7 @@ stimuli: - name: Correct Client Construction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Service clients are constructed using the ::new() constructor with an endpoint string, credential, and Option parameter. @@ -172,7 +171,7 @@ stimuli: #[allow(clippy::needless_update)] annotation is used. - name: Pager and Stream Usage type: prompt - model: gpt-5.3-codex + model: claude-sonnet-4.6 weight: 1.0 prompt: > List operations return and consume azure_core::Pager using @@ -186,7 +185,7 @@ stimuli: - name: Response Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Calls to service clients correctly handle Response. When service methods return a model type, code calls into_model() to deserialize @@ -196,7 +195,7 @@ stimuli: - name: Long-Running Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Long-running operations use azure_core::Poller correctly. If a client calls into an API returning a Poller, code awaits @@ -207,7 +206,7 @@ stimuli: - name: Struct Initialization with Default type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > If a client initializes a structure for SDK types (client options, method options, model types) the code uses ..Default::default() to protect @@ -215,7 +214,7 @@ stimuli: - name: Ownership and Borrowing type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Code uses references (&str, &T) for borrowed data and avoids unnecessary .clone() calls. Credentials are shared via Arc or @@ -231,46 +230,3 @@ stimuli: - "+nightly" - "-Zscript" - ".vally/tools/check-no-secrets.rs" - - name: Rust Cargo Trajectory Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - collect_trajectory_compiler_errors: true - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Build Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo build - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Check Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo check - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail - - name: Post prompt Rust Cargo Clippy Failure - type: rust-cargo-build-failure-check - weight: 1.0 - config: - commands: - - command: cargo clippy - emit_in_metadata: true - error_exceptions: - - error_code: E0277 - message_pattern: "Option" - action: ignore_for_fail diff --git a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml index 859e46e3..42bebb73 100644 --- a/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml +++ b/tests/scenarios/azure-storage-blob-rust/vally/skill_effectiveness_experiment.yaml @@ -2,8 +2,6 @@ name: azure-storage-blob-rust-skill-experiment evals: - skill_effectiveness_eval.yaml - - eval-nobuild-experiment.yaml - - eval-keyvault-encryption.yaml vary: - /defaults/model diff --git a/tests/scenarios/azure-storage-file-datalake-py/scenarios.yaml b/tests/scenarios/azure-storage-file-datalake-py/scenarios.yaml index 4c86bd93..82b9eaba 100644 --- a/tests/scenarios/azure-storage-file-datalake-py/scenarios.yaml +++ b/tests/scenarios/azure-storage-file-datalake-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-storage-file-datalake-py/vally/eval.yaml b/tests/scenarios/azure-storage-file-datalake-py/vally/eval.yaml new file mode 100644 index 00000000..e79a127e --- /dev/null +++ b/tests/scenarios/azure-storage-file-datalake-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: azure-storage-file-datalake-py-eval +description: Vally evaluation suite for azure-storage-file-datalake-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_service_client_and_filesystem + prompt: |- + Create a basic example that authenticates with DefaultAzureCredential, + creates a DataLakeServiceClient, and creates/gets a file system. + Include listing file systems. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: directory_operations + prompt: |- + Show how to create a directory, get a directory client, + create a subdirectory, and rename the directory. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-file-datalake-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..586dabfe --- /dev/null +++ b/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,139 @@ +name: azure-storage-file-datalake-py-eval +description: Vally evaluation suite for azure-storage-file-datalake-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_service_client_and_filesystem + prompt: |- + Create a basic example that authenticates with DefaultAzureCredential, + creates a DataLakeServiceClient, and creates/gets a file system. + Include listing file systems. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: directory_operations + prompt: |- + Show how to create a directory, get a directory client, + create a subdirectory, and rename the directory. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-file-datalake-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..7ce7618d --- /dev/null +++ b/tests/scenarios/azure-storage-file-datalake-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-storage-file-datalake-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-datalake-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-file-share-py/scenarios.yaml b/tests/scenarios/azure-storage-file-share-py/scenarios.yaml index 1c518d79..f92a3111 100644 --- a/tests/scenarios/azure-storage-file-share-py/scenarios.yaml +++ b/tests/scenarios/azure-storage-file-share-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-storage-file-share-py/vally/eval.yaml b/tests/scenarios/azure-storage-file-share-py/vally/eval.yaml new file mode 100644 index 00000000..bb66ccbb --- /dev/null +++ b/tests/scenarios/azure-storage-file-share-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-storage-file-share-py-eval +description: Vally evaluation suite for azure-storage-file-share-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: service_client_connection_string + prompt: |- + Create a ShareServiceClient using a connection string and demonstrate + creating a share, listing shares, and deleting the share. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: service_client_default_azure_credential + prompt: |- + Use DefaultAzureCredential with a ShareServiceClient and account_url. + Create a share client, create a directory, and clean up the share. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-file-share-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d8533d2f --- /dev/null +++ b/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-storage-file-share-py-eval +description: Vally evaluation suite for azure-storage-file-share-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: service_client_connection_string + prompt: |- + Create a ShareServiceClient using a connection string and demonstrate + creating a share, listing shares, and deleting the share. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: service_client_default_azure_credential + prompt: |- + Use DefaultAzureCredential with a ShareServiceClient and account_url. + Create a share client, create a directory, and clean up the share. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-file-share-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..d6d9022b --- /dev/null +++ b/tests/scenarios/azure-storage-file-share-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-storage-file-share-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-file-share-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-queue-py/scenarios.yaml b/tests/scenarios/azure-storage-queue-py/scenarios.yaml index 848489f5..fca23d17 100644 --- a/tests/scenarios/azure-storage-queue-py/scenarios.yaml +++ b/tests/scenarios/azure-storage-queue-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/azure-storage-queue-py/vally/eval.yaml b/tests/scenarios/azure-storage-queue-py/vally/eval.yaml new file mode 100644 index 00000000..1b961036 --- /dev/null +++ b/tests/scenarios/azure-storage-queue-py/vally/eval.yaml @@ -0,0 +1,136 @@ +name: azure-storage-queue-py-eval +description: Vally evaluation suite for azure-storage-queue-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_queue_service_client + prompt: |- + Create a basic example using QueueServiceClient with DefaultAzureCredential. + Create a queue, list queues, and delete the queue. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: queue_client_send_receive_peek + prompt: |- + Create a QueueClient that sends a message, receives messages, + peeks messages, and deletes messages after processing. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-queue-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-queue-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d98d1a52 --- /dev/null +++ b/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: azure-storage-queue-py-eval +description: Vally evaluation suite for azure-storage-queue-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_queue_service_client + prompt: |- + Create a basic example using QueueServiceClient with DefaultAzureCredential. + Create a queue, list queues, and delete the queue. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: queue_client_send_receive_peek + prompt: |- + Create a QueueClient that sends a message, receives messages, + peeks messages, and deletes messages after processing. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap credential/client initialization and all storage operations in try/except blocks + * Handle: ResourceNotFoundError, ResourceExistsError, AzureError + * Example: try: blob_client.upload_blob(data) except ResourceExistsError: pass + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - azure-storage-queue-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-queue-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..d162882c --- /dev/null +++ b/tests/scenarios/azure-storage-queue-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: azure-storage-queue-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/azure-storage-queue-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml b/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml deleted file mode 100644 index a48e09f8..00000000 --- a/tests/scenarios/azure-storage-queue-rust/vally/eval-experiment.yaml +++ /dev/null @@ -1,165 +0,0 @@ -name: azure-storage-queue-rust-eval -description: Evaluation suite for azure-storage-queue-rust -version: "1.0" -type: capability -scoring: - weights: {} - threshold: 1.0 -defaults: - runs: 1 - timeout: 30m - model: claude-sonnet-4.6 -stimuli: - - name: rust-queue-operations - prompt: | - Write a Rust program named rust-queue-ops that performs the following queue storage operations: - 1. Create a QueueServiceClient from an endpoint URL with TokenCredential authentication - 2. Create or get a queue client for a queue named "work-queue" - 3. Send 3 messages to the queue with different message text - 4. Receive messages from the queue (up to 5 messages) - 5. Peek at messages in the queue without removing them - 6. For each received message, extract the message ID, pop receipt, and message text - 7. Delete messages from the queue using message ID and pop receipt - 8. Demonstrate proper error handling for queue operations - - The program should compile and run successfully with the official Azure Storage Queue SDK, - demonstrating full CRUD operations on queue messages with proper TokenCredential-based authentication. - - Ensure the following SDK patterns are demonstrated: - - QueueServiceClient initialization with endpoint and credential - - QueueClient derivation from QueueServiceClient - - Message send/receive/peek/delete workflows - - Proper use of try_into() for message models - - Error handling with Result-based patterns - rubric: - - "Implements complete queue message lifecycle: send, receive, peek, and delete operations end-to-end." - - "Uses official azure_storage_queue SDK with TokenCredential-based authentication." - - "Uses async Rust with Tokio runtime and idiomatic Result-based error propagation." - - "Code compiles and passes cargo check." - - "Correctly derives QueueClient from QueueServiceClient and uses queue_client() method." - - "Message receive operations properly extract message_id, pop_receipt, and message_text fields." - - "Message delete operations use both message_id and pop_receipt from received messages." - - "Peek operations demonstrate reading messages without consuming them from the queue." - - "Avoids hardcoded credentials in source code." - workspace_files: - include: - - "**/Cargo.toml" - - "**/Cargo.lock" - - "**/build.rs" - - "**/src/**/*.rs" - - "**/examples/**/*.rs" - - "**/tests/**/*.rs" - exclude: - - "**/target/**" - - "**/*.rlib" - - "**/*.rmeta" - - "**/*.d" - - "**/*.o" - - "**/*.obj" - - "**/*.pdb" - - "**/*.exe" - - "**/*.dll" - - "**/*.so" - - "**/*.dylib" - - "**/*.a" - - "**/*.lib" - - "**/*.wasm" - environment: - files: - - src: workspace/cargo.toml - dest: Cargo.toml - constraints: - expect_skills: - - azure-storage-queue-rust - graders: - - type: prompt - name: "Queue operations completeness" - config: - model: gpt-5.3-codex - prompt: | - - QueueServiceClient initialization with endpoint URL and credential - - QueueClient derived from service client - - Send message operations with message_text - - Receive messages with proper message_id, pop_receipt extraction - - Peek messages without removal - - Delete messages using message_id and pop_receipt pair - - Error handling for queue operations - scoring: scale_1_5 - - - type: run-command - name: "Build Application" - config: - command: cargo - args: - - "build" - timeout: 120s - stderr_contains: "Finished" - - type: run-command - name: "Check Application" - config: - command: cargo - args: - - "check" - timeout: 90s - - name: Rust Clippy Lints - type: run-command - weight: 1.0 - config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s - - name: Official Azure SDK Crate Selection - type: program - weight: 1.0 - config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" - - name: TokenCredential Authentication - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Authentication uses TokenCredential from azure_identity, created via - DeveloperToolsCredential::new(None) or ManagedIdentityCredential::new(None). - The credential is passed to QueueServiceClient::new() as Some(credential). - For production, uses ManagedIdentityCredential; for local dev, uses DeveloperToolsCredential. - - name: QueueServiceClient Construction - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - QueueServiceClient is created via QueueServiceClient::new() with three parameters: - service_url (Url type), Some(credential), and None for options. - QueueClient is derived via service_client.queue_client("")? - - name: Queue Message Operations - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Send messages via queue_client.send_message(message.try_into()?, None).await?. - Receive messages via queue_client.receive_messages(None).await? returning a response - that converts to model with items containing message_id, pop_receipt, and message_text fields. - Delete messages via queue_client.delete_message(id, pop_receipt, None).await?. - Peek via queue_client.peek_messages(None).await?. - - name: Async Runtime - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Code uses #[tokio::main] and async/await. Queue operations are properly awaited. - Result> main signature for error propagation. - - name: Error Handling - type: prompt - weight: 1.0 - model: gpt-5.3-codex - prompt: > - Queue operations are wrapped in Result-based error handling using ? operator. - No .unwrap() calls except where failure is impossible. - Main function returns Result for proper error propagation. diff --git a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml index 131b8112..4d5c5aa7 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-queue-operations prompt: | @@ -41,7 +42,7 @@ stimuli: - "Message delete operations use both message_id and pop_receipt from received messages." - "Peek operations demonstrate reading messages without consuming them from the queue." - "Avoids hardcoded credentials in source code." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -68,6 +69,16 @@ stimuli: skills: - ../../../../.github/plugins/azure-sdk-rust/skills/azure-storage-queue-rust files: + - src: ../../_shared/vally/tools/find-cargo-root.js + dest: .vally/tools/find-cargo-root.js + - src: ../../_shared/vally/tools/check-azure-crates.rs + dest: .vally/tools/check-azure-crates.rs + - src: ../../_shared/vally/tools/check-token-credential.rs + dest: .vally/tools/check-token-credential.rs + - src: ../../_shared/vally/tools/check-async-runtime.rs + dest: .vally/tools/check-async-runtime.rs + - src: ../../_shared/vally/tools/check-no-secrets.rs + dest: .vally/tools/check-no-secrets.rs - src: workspace/cargo.toml dest: Cargo.toml constraints: @@ -77,7 +88,6 @@ stimuli: - type: prompt name: "Queue operations completeness" config: - model: gpt-5.3-codex prompt: | - QueueServiceClient initialization with endpoint URL and credential - QueueClient derived from service client @@ -88,73 +98,67 @@ stimuli: - Error handling for queue operations scoring: scale_1_5 - - type: run-command - name: "Build Application" + - name: "Build Application" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "build" - timeout: 120s - stderr_contains: "Finished" - - type: run-command - name: "Check Application" + commands: + - command: cargo build + emit_in_metadata: true + - name: "Check Application" + type: rust-cargo-build-failure-check config: - command: cargo - args: - - "check" - timeout: 90s + commands: + - command: cargo check + emit_in_metadata: true - name: Rust Clippy Lints - type: run-command + type: rust-cargo-build-failure-check weight: 1.0 config: - command: cargo - args: - - "clippy" - - "--" - - "-D" - - "warnings" - timeout: 120s + commands: + - command: cargo clippy -- -D warnings + emit_in_metadata: true - name: Official Azure SDK Crate Selection - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-azure-crates.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-azure-crates.rs + emit_in_metadata: true - name: TokenCredential Authentication - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-token-credential.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-token-credential.rs + emit_in_metadata: true - name: Async-First with Tokio Runtime - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-async-runtime.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-async-runtime.rs + emit_in_metadata: true - name: No Hardcoded Secrets - type: program + type: rust-cargo-build-failure-check weight: 1.0 config: - program: cargo - args: - - "+nightly" - - "-Zscript" - - ".vally/tools/check-no-secrets.rs" + commands: + - command: cargo +nightly -Zscript .vally/tools/check-no-secrets.rs + emit_in_metadata: true + - name: Rust Cargo Trajectory Build Failure + type: rust-cargo-build-failure-check + weight: 1.0 + config: + collect_trajectory_compiler_errors: true + emit_in_metadata: true + error_exceptions: + - error_code: E0277 + message_pattern: "Option" + action: ignore_for_fail - name: QueueServiceClient Construction type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > QueueServiceClient is created via QueueServiceClient::new() with three parameters: service_url (Url type), Some(credential), and None for options. @@ -162,7 +166,6 @@ stimuli: - name: Queue Message Operations type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Send messages via queue_client.send_message(message.try_into()?, None).await?. Receive messages via queue_client.receive_messages(None).await? returning a response @@ -172,7 +175,6 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex prompt: > Queue operations are wrapped in Result-based error handling using ? operator. No .unwrap() calls except where failure is impossible. diff --git a/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml b/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml index f4ae4478..78ad4708 100644 --- a/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml +++ b/tests/scenarios/azure-storage-queue-rust/vally/skill_effectiveness_eval.yaml @@ -7,8 +7,9 @@ scoring: threshold: 0.75 defaults: runs: 1 - timeout: 30m + timeout: 60m model: claude-sonnet-4.6 + grader_model: gpt-5.5 stimuli: - name: rust-queue-operations prompt: | @@ -41,7 +42,7 @@ stimuli: - "Message delete operations use both message_id and pop_receipt from received messages." - "Peek operations demonstrate reading messages without consuming them from the queue." - "Avoids hardcoded credentials in source code." - workspace_files: + artifacts: include: - "**/Cargo.toml" - "**/Cargo.lock" @@ -83,7 +84,7 @@ stimuli: - type: prompt name: "Queue operations completeness" config: - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: | - QueueServiceClient initialization with endpoint URL and credential - QueueClient derived from service client @@ -159,7 +160,7 @@ stimuli: - name: QueueServiceClient Construction type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > QueueServiceClient is created via QueueServiceClient::new() with three parameters: service_url (Url type), Some(credential), and None for options. @@ -167,7 +168,7 @@ stimuli: - name: Queue Message Operations type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Send messages via queue_client.send_message(message.try_into()?, None).await?. Receive messages via queue_client.receive_messages(None).await? returning a response @@ -177,7 +178,7 @@ stimuli: - name: Error Handling type: prompt weight: 1.0 - model: gpt-5.3-codex + model: claude-sonnet-4.6 prompt: > Queue operations are wrapped in Result-based error handling using ? operator. No .unwrap() calls except where failure is impossible. diff --git a/tests/scenarios/fastapi-router-py/scenarios.yaml b/tests/scenarios/fastapi-router-py/scenarios.yaml index d4fa0ae5..8d9efbe2 100644 --- a/tests/scenarios/fastapi-router-py/scenarios.yaml +++ b/tests/scenarios/fastapi-router-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific FastAPI router pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 @@ -36,16 +36,16 @@ scenarios: mock_response: | from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query, status - + from app.models.project import ProjectResponse, ProjectCreate, ProjectUpdate from app.services.project_service import ProjectService - + router = APIRouter(prefix="/api", tags=["projects"]) - + def get_service() -> ProjectService: """Dependency to get service instance.""" return ProjectService() - + @router.get("/projects", response_model=list[ProjectResponse]) async def list_projects( limit: int = Query(default=50, ge=1, le=100), @@ -54,7 +54,7 @@ scenarios: ) -> list[ProjectResponse]: """List all projects with pagination.""" return await service.list_projects(limit=limit, offset=offset) - + @router.get("/projects/{project_id}", response_model=ProjectResponse) async def get_project( project_id: str, @@ -68,7 +68,7 @@ scenarios: detail="Project not found", ) return result - + @router.post( "/projects", response_model=ProjectResponse, @@ -80,7 +80,7 @@ scenarios: ) -> ProjectResponse: """Create a new project.""" return await service.create_project(data) - + @router.patch("/projects/{project_id}", response_model=ProjectResponse) async def update_project( project_id: str, @@ -95,7 +95,7 @@ scenarios: detail="Project not found", ) return await service.update_project(project_id, data) - + @router.delete( "/projects/{project_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -130,27 +130,27 @@ scenarios: - "HTTP_401_UNAUTHORIZED" - "HTTP_403_FORBIDDEN" forbidden_patterns: - - "current_user: str" # Should be User type - - "current_user: Optional[^\\[]" # Should have full type like Optional[User] - - "get_current_user_required.*Optional" # Can't be both required and optional + - "current_user: str" # Should be User type + - "current_user: Optional[^\\[]" # Should have full type like Optional[User] + - "get_current_user_required.*Optional" # Can't be both required and optional tags: - authentication - security mock_response: | from typing import Optional from fastapi import APIRouter, Depends, HTTPException, status - + from app.auth.jwt import get_current_user, get_current_user_required from app.models.user import User from app.models.task import TaskResponse, TaskCreate, TaskUpdate from app.services.task_service import TaskService - + router = APIRouter(prefix="/api", tags=["tasks"]) - + def get_service() -> TaskService: """Dependency to get service instance.""" return TaskService() - + @router.get("/tasks", response_model=list[TaskResponse]) async def list_tasks( current_user: Optional[User] = Depends(get_current_user), @@ -159,7 +159,7 @@ scenarios: """List tasks. Authentication optional.""" user_id = current_user.id if current_user else None return await service.list_tasks(user_id=user_id) - + @router.post( "/tasks", response_model=TaskResponse, @@ -172,7 +172,7 @@ scenarios: ) -> TaskResponse: """Create a new task. Authentication required.""" return await service.create_task(data, owner_id=current_user.id) - + @router.patch("/tasks/{task_id}", response_model=TaskResponse) async def update_task( task_id: str, @@ -193,7 +193,7 @@ scenarios: detail="Not authorized to update this task", ) return await service.update_task(task_id, data) - + @router.delete( "/tasks/{task_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -237,7 +237,7 @@ scenarios: forbidden_patterns: - "response_model=dict" - "response_model=str" - - "response_model=list[^\\[]" # Should specify type: list[Model], not just list + - "response_model=list[^\\[]" # Should specify type: list[Model], not just list - "-> dict:" - "-> Any:" tags: @@ -246,22 +246,22 @@ scenarios: mock_response: | from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel - + from app.models.article import ArticleResponse, ArticleCreate from app.services.article_service import ArticleService - + router = APIRouter(prefix="/api", tags=["articles"]) - + def get_service() -> ArticleService: return ArticleService() - + @router.get("/articles", response_model=list[ArticleResponse]) async def list_articles( service: ArticleService = Depends(get_service), ) -> list[ArticleResponse]: """List all articles.""" return await service.list_articles() - + @router.get("/articles/{article_id}", response_model=ArticleResponse) async def get_article( article_id: str, @@ -275,7 +275,7 @@ scenarios: detail="Article not found", ) return article - + @router.post( "/articles", response_model=ArticleResponse, @@ -287,7 +287,7 @@ scenarios: ) -> ArticleResponse: """Create a new article.""" return await service.create_article(data) - + @router.delete( "/articles/{article_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -323,42 +323,52 @@ scenarios: - "Query\\(default=" - "description=" forbidden_patterns: - - "q: str" # Should have Query validation - - "limit: int = 50" # Should have Query with constraints - - "offset: int = 0" # Should have Query with constraints - - "Query\\(\\)" # Query should have parameters + - "limit: int = 50" # Should have Query with constraints + - "offset: int = 0" # Should have Query with constraints + - "Query\\(\\)" # Query should have parameters tags: - validation - query-parameters mock_response: | - from typing import Annotated, Optional - from fastapi import APIRouter, Depends, Query, status - - from app.models.search import SearchResponse - from app.services.search_service import SearchService - - router = APIRouter(prefix="/api", tags=["search"]) - - def get_service() -> SearchService: - return SearchService() - - @router.get("/search", response_model=list[SearchResponse]) + from typing import Literal + + from fastapi import APIRouter, Query + + router = APIRouter(prefix="/search", tags=["search"]) + + + @router.get("") async def search( - q: Annotated[str, Query(min_length=1, max_length=200, description="Search query")], - limit: Annotated[int, Query(ge=1, le=100, description="Max results")] = 50, - offset: Annotated[int, Query(ge=0, description="Results to skip")] = 0, - sort_by: Annotated[Optional[str], Query(description="Sort by: relevance, date, popularity")] = "relevance", - service: SearchService = Depends(get_service), - ) -> list[SearchResponse]: - """ - Search for items. - - - **q**: Search term (required, 1-200 characters) - - **limit**: Maximum results to return (1-100, default 50) - - **offset**: Number of results to skip (default 0) - - **sort_by**: Sort results by relevance, date, or popularity - """ - return await service.search(q=q, limit=limit, offset=offset, sort_by=sort_by) + q: str = Query( + ..., + min_length=1, + max_length=200, + description="Search term to query for.", + ), + limit: int = Query( + 50, + ge=1, + le=100, + description="Maximum number of results to return.", + ), + offset: int = Query( + 0, + ge=0, + description="Number of results to skip before returning matches.", + ), + sort_by: Literal["relevance", "date", "popularity"] = Query( + "relevance", + description="Sort order for search results.", + ), + ) -> dict[str, object]: + """Search for items matching the provided query parameters.""" + return { + "q": q, + "limit": limit, + "offset": offset, + "sort_by": sort_by, + "results": [], + } # Error Handling with HTTPException - name: error_handling_http_exception @@ -377,25 +387,25 @@ scenarios: - "if.*is None" - "if.*owner_id" forbidden_patterns: - - "except" # Should use HTTPException, not try/except - - "return None" # Should raise HTTPException instead - - "raise Exception" # Should use HTTPException + - "except" # Should use HTTPException, not try/except + - "return None" # Should raise HTTPException instead + - "raise Exception" # Should use HTTPException tags: - error-handling - security mock_response: | from fastapi import APIRouter, Depends, HTTPException, status - + from app.models.user import UserResponse, UserUpdate from app.auth.jwt import get_current_user_required from app.models.user import User from app.services.user_service import UserService - + router = APIRouter(prefix="/api", tags=["users"]) - + def get_service() -> UserService: return UserService() - + @router.get("/users/{user_id}", response_model=UserResponse) async def get_user( user_id: str, @@ -409,7 +419,7 @@ scenarios: detail="User not found", ) return user - + @router.patch("/users/{user_id}", response_model=UserResponse) async def update_user( user_id: str, @@ -456,28 +466,28 @@ scenarios: - "posts" - "comments" forbidden_patterns: - - "include_router.*prefix.*prefix" # Double prefix mounting - - "APIRouter\\(\\)" # Should have prefix + - "include_router.*prefix.*prefix" # Double prefix mounting + - "APIRouter\\(\\)" # Should have prefix tags: - routing - advanced mock_response: | # routers/posts.py from fastapi import APIRouter, Depends, HTTPException, status - + from app.models.post import PostResponse, PostCreate from app.services.post_service import PostService - + router = APIRouter(prefix="/api/posts", tags=["posts"]) - + def get_service() -> PostService: return PostService() - + @router.get("", response_model=list[PostResponse]) async def list_posts(service: PostService = Depends(get_service)) -> list[PostResponse]: """List all posts.""" return await service.list_posts() - + @router.post( "", response_model=PostResponse, @@ -489,23 +499,23 @@ scenarios: ) -> PostResponse: """Create a new post.""" return await service.create_post(data) - + # routers/comments.py from fastapi import APIRouter, Depends, HTTPException, status - + from app.models.comment import CommentResponse, CommentCreate from app.services.comment_service import CommentService - + router = APIRouter(prefix="/api/comments", tags=["comments"]) - + def get_service() -> CommentService: return CommentService() - + @router.get("", response_model=list[CommentResponse]) async def list_comments(service: CommentService = Depends(get_service)) -> list[CommentResponse]: """List all comments.""" return await service.list_comments() - + @router.post( "", response_model=CommentResponse, @@ -517,12 +527,12 @@ scenarios: ) -> CommentResponse: """Create a new comment.""" return await service.create_comment(data) - + # main.py from fastapi import FastAPI from routers.posts import router as posts_router from routers.comments import router as comments_router - + app = FastAPI(title="My API") app.include_router(posts_router) app.include_router(comments_router) @@ -543,32 +553,32 @@ scenarios: - "NotificationService" - "HTTPException" forbidden_patterns: - - "(? NotificationService: """Dependency to get service instance.""" return NotificationService() - + @router.get("/notifications", response_model=list[NotificationResponse]) async def list_notifications( service: NotificationService = Depends(get_service), ) -> list[NotificationResponse]: """List all notifications.""" return await service.list_notifications() - + @router.get("/notifications/{notification_id}", response_model=NotificationResponse) async def get_notification( notification_id: str, @@ -582,7 +592,7 @@ scenarios: detail="Notification not found", ) return notification - + @router.post( "/notifications", response_model=NotificationResponse, @@ -594,7 +604,7 @@ scenarios: ) -> NotificationResponse: """Create a new notification.""" return await service.create_notification(data) - + @router.delete( "/notifications/{notification_id}", status_code=status.HTTP_204_NO_CONTENT, @@ -629,26 +639,26 @@ scenarios: - "service.*=.*Depends" - "current_user.*=.*Depends" forbidden_patterns: - - "service = ReportService\\(\\)" # Should use Depends - - "current_user: str" # Should be User type + - "service = ReportService\\(\\)" # Should use Depends + - "current_user: str" # Should be User type tags: - dependencies - advanced mock_response: | from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query, status - + from app.auth.jwt import get_current_user_required from app.models.user import User from app.models.report import ReportResponse, ReportCreate from app.services.report_service import ReportService - + router = APIRouter(prefix="/api", tags=["reports"]) - + def get_service() -> ReportService: """Dependency to get service instance.""" return ReportService() - + @router.get("/reports", response_model=list[ReportResponse]) async def list_reports( limit: int = Query(default=50, ge=1, le=100), @@ -664,7 +674,7 @@ scenarios: offset=offset, status_filter=status_filter, ) - + @router.post( "/reports", response_model=ReportResponse, @@ -677,7 +687,7 @@ scenarios: ) -> ReportResponse: """Create a new report for the current user.""" return await service.create_report(data, user_id=current_user.id) - + @router.get("/reports/{report_id}", response_model=ReportResponse) async def get_report( report_id: str, diff --git a/tests/scenarios/fastapi-router-py/vally/eval.yaml b/tests/scenarios/fastapi-router-py/vally/eval.yaml new file mode 100644 index 00000000..73efafdc --- /dev/null +++ b/tests/scenarios/fastapi-router-py/vally/eval.yaml @@ -0,0 +1,158 @@ +name: fastapi-router-py-eval +description: Vally evaluation suite for fastapi-router-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_crud_router + prompt: |- + Create a FastAPI router for managing "projects" with full CRUD operations. + Include GET (list and single), POST (create), PATCH (update), and DELETE endpoints. + Use proper status codes, response models, and async/await patterns. + Include pagination with limit and offset query parameters. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "project" (snake_case), "Project" (PascalCase), "projects" (plural) + * NEVER use {{variable}}, {{resource_name}}, {{ResourceName}}, or {{resource_plural}} + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling: + * Wrap async endpoint logic in try/except blocks + * Handle HTTPException, ValueError, and generic Exception types + * Provide meaningful error messages in responses + * Example: try: ... except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: router_with_authentication + prompt: |- + Create a FastAPI router for "tasks" where: + - GET endpoints (list, single) allow optional authentication + - POST, PATCH, DELETE endpoints require authentication + - Use get_current_user for optional and get_current_user_required for required auth + Include proper type hints and HTTPException for unauthorized access. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "task" (snake_case), "Task" (PascalCase), "tasks" (plural) + * NEVER use {{variable}}, {{resource_name}}, {{ResourceName}}, or {{resource_plural}} + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling: + * Wrap async endpoint logic in try/except blocks + * Handle HTTPException, ValueError, and generic Exception types + * Provide meaningful error messages in responses + * Example: try: ... except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - fastapi-router-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/fastapi-router-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..a5c3a5ff --- /dev/null +++ b/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,159 @@ +name: fastapi-router-py-eval +description: Vally evaluation suite for fastapi-router-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: basic_crud_router + prompt: |- + Create a FastAPI router for managing "projects" with full CRUD operations. + Include GET (list and single), POST (create), PATCH (update), and DELETE endpoints. + Use proper status codes, response models, and async/await patterns. + Include pagination with limit and offset query parameters. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "project" (snake_case), "Project" (PascalCase), "projects" (plural) + * NEVER use {{variable}}, {{resource_name}}, {{ResourceName}}, or {{resource_plural}} + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling: + * Wrap async endpoint logic in try/except blocks + * Handle HTTPException, ValueError, and generic Exception types + * Provide meaningful error messages in responses + * Example: try: ... except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: router_with_authentication + prompt: |- + Create a FastAPI router for "tasks" where: + - GET endpoints (list, single) allow optional authentication + - POST, PATCH, DELETE endpoints require authentication + - Use get_current_user for optional and get_current_user_required for required auth + Include proper type hints and HTTPException for unauthorized access. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "task" (snake_case), "Task" (PascalCase), "tasks" (plural) + * NEVER use {{variable}}, {{resource_name}}, {{ResourceName}}, or {{resource_plural}} + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling: + * Wrap async endpoint logic in try/except blocks + * Handle HTTPException, ValueError, and generic Exception types + * Provide meaningful error messages in responses + * Example: try: ... except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - fastapi-router-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/fastapi-router-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..abf441ec --- /dev/null +++ b/tests/scenarios/fastapi-router-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: fastapi-router-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/fastapi-router-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/fastapi-router-py/vally/syntax-check-config.json b/tests/scenarios/fastapi-router-py/vally/syntax-check-config.json new file mode 100644 index 00000000..4b941664 --- /dev/null +++ b/tests/scenarios/fastapi-router-py/vally/syntax-check-config.json @@ -0,0 +1,6 @@ +{ + "excludePatterns": [ + "*/assets/template.py", + "**/_**/template.py" + ] +} \ No newline at end of file diff --git a/tests/scenarios/m365-agents-py/scenarios.yaml b/tests/scenarios/m365-agents-py/scenarios.yaml index 40d0aaa1..25e9e4b3 100644 --- a/tests/scenarios/m365-agents-py/scenarios.yaml +++ b/tests/scenarios/m365-agents-py/scenarios.yaml @@ -3,7 +3,7 @@ # Each scenario tests a specific usage pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/m365-agents-py/vally/eval.yaml b/tests/scenarios/m365-agents-py/vally/eval.yaml new file mode 100644 index 00000000..8a642e90 --- /dev/null +++ b/tests/scenarios/m365-agents-py/vally/eval.yaml @@ -0,0 +1,137 @@ +name: m365-agents-py-eval +description: Vally evaluation suite for m365-agents-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: aiohttp_agent_hosting + prompt: |- + Create a minimal aiohttp-hosted Microsoft 365 agent in Python using + AgentApplication, start_agent_process, and jwt_authorization_middleware. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap AgentApplication initialization and turn/message handling in try/except blocks + * Handle: AuthenticationError, ServiceError, generic Exception + * Example: try: await app.process_activity(req) except Exception as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: agent_application_routing + prompt: |- + Create an AgentApplication that handles welcome messages with + conversation_update, message handlers with regex, and an error handler. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap AgentApplication initialization and turn/message handling in try/except blocks + * Handle: AuthenticationError, ServiceError, generic Exception + * Example: try: await app.process_activity(req) except Exception as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - m365-agents-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/m365-agents-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/m365-agents-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/m365-agents-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..a434d054 --- /dev/null +++ b/tests/scenarios/m365-agents-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,138 @@ +name: m365-agents-py-eval +description: Vally evaluation suite for m365-agents-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: aiohttp_agent_hosting + prompt: |- + Create a minimal aiohttp-hosted Microsoft 365 agent in Python using + AgentApplication, start_agent_process, and jwt_authorization_middleware. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap AgentApplication initialization and turn/message handling in try/except blocks + * Handle: AuthenticationError, ServiceError, generic Exception + * Example: try: await app.process_activity(req) except Exception as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: agent_application_routing + prompt: |- + Create an AgentApplication that handles welcome messages with + conversation_update, message handlers with regex, and an error handler. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Include explicit exception handling: + * Wrap AgentApplication initialization and turn/message handling in try/except blocks + * Handle: AuthenticationError, ServiceError, generic Exception + * Example: try: await app.process_activity(req) except Exception as e: print(f'Error: {e}') + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - m365-agents-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/m365-agents-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/m365-agents-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/m365-agents-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..bb656e5c --- /dev/null +++ b/tests/scenarios/m365-agents-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: m365-agents-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/m365-agents-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/pydantic-models-py/scenarios.yaml b/tests/scenarios/pydantic-models-py/scenarios.yaml index 0a10ccb2..1af0cde2 100644 --- a/tests/scenarios/pydantic-models-py/scenarios.yaml +++ b/tests/scenarios/pydantic-models-py/scenarios.yaml @@ -2,7 +2,7 @@ # Each scenario tests a specific Pydantic model pattern against acceptance criteria config: - model: gpt-4 + model: gpt-5.5 max_tokens: 2000 temperature: 0.3 diff --git a/tests/scenarios/pydantic-models-py/vally/eval.yaml b/tests/scenarios/pydantic-models-py/vally/eval.yaml new file mode 100644 index 00000000..f3daa2c9 --- /dev/null +++ b/tests/scenarios/pydantic-models-py/vally/eval.yaml @@ -0,0 +1,155 @@ +name: pydantic-models-py-eval +description: Vally evaluation suite for pydantic-models-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: base_create_response_indb + prompt: |- + Create a complete set of Pydantic models for a "Document" resource following the + multi-model pattern. Include Base, Create, Update, Response, and InDB models. + Use Field constraints, proper inheritance, and camelCase aliases with populate_by_name. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "Document" (PascalCase), "document" (snake_case) + * NEVER use {{variable}}, {{ResourceName}}, {{resource_name}}, or similar placeholders + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling in any code that instantiates or validates models: + * Wrap model instantiation in try/except blocks + * Handle ValidationError from pydantic + * Example: try: Document(...) except ValidationError as e: handle_validation_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: camel_case_aliases + prompt: |- + Create a Pydantic model for a "Project" resource with snake_case fields but camelCase + aliases for API responses. Include created_at and updated_at with aliases. + Ensure populate_by_name is True to accept both formats. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "Project" (PascalCase), "project" (snake_case) + * NEVER use {{variable}}, {{ResourceName}}, {{resource_name}}, or similar placeholders + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling in any code that instantiates or validates models: + * Wrap model instantiation in try/except blocks + * Handle ValidationError from pydantic + * Example: try: Project(...) except ValidationError as e: handle_validation_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - pydantic-models-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/pydantic-models-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" + diff --git a/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_eval.yaml b/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_eval.yaml new file mode 100644 index 00000000..d35001a5 --- /dev/null +++ b/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_eval.yaml @@ -0,0 +1,156 @@ +name: pydantic-models-py-eval +description: Vally evaluation suite for pydantic-models-py +version: "1.0" +type: capability +scoring: + weights: {} + threshold: 0.75 +defaults: + runs: 1 + timeout: 30m + model: claude-sonnet-4.6 + grader_model: gpt-5.5 +stimuli: + - name: base_create_response_indb + prompt: |- + Create a complete set of Pydantic models for a "Document" resource following the + multi-model pattern. Include Base, Create, Update, Response, and InDB models. + Use Field constraints, proper inheritance, and camelCase aliases with populate_by_name. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "Document" (PascalCase), "document" (snake_case) + * NEVER use {{variable}}, {{ResourceName}}, {{resource_name}}, or similar placeholders + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling in any code that instantiates or validates models: + * Wrap model instantiation in try/except blocks + * Handle ValidationError from pydantic + * Example: try: Document(...) except ValidationError as e: handle_validation_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 + - name: camel_case_aliases + prompt: |- + Create a Pydantic model for a "Project" resource with snake_case fields but camelCase + aliases for API responses. Include created_at and updated_at with aliases. + Ensure populate_by_name is True to accept both formats. + + Output requirements for grading: + - Generate Python code, not prose. + - Create at least one runnable Python file in the workspace (for example, `main.py`). + - Return the final answer as Python code only. + - CRITICAL: Use ONLY concrete resource names. Substitute all placeholders: + * Use "Project" (PascalCase), "project" (snake_case) + * NEVER use {{variable}}, {{ResourceName}}, {{resource_name}}, or similar placeholders + - All .py files must pass Python syntax validation: `python -m py_compile ` + - Do NOT reference, import, or copy from template.py files + - CRITICAL: Include explicit exception handling in any code that instantiates or validates models: + * Wrap model instantiation in try/except blocks + * Handle ValidationError from pydantic + * Example: try: Project(...) except ValidationError as e: handle_validation_error(e) + rubric: + - Uses the APIs, imports, and workflow expected by the skill. + - Keeps credentials in environment variables and avoids hardcoded secrets. + - Provides complete executable Python code with clear control flow and error handling. + - Includes explicit try/except exception handling. + - Uses modern Python conventions and readable structure. + graders: + - type: run-command + name: Python syntax check + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-syntax.mjs + - .vally/syntax-check-config.json + timeout: 90s + - type: run-command + name: Python idiomatic static checks + weight: 1 + config: + command: node + args: + - .vally/tools/check-python-idiomatic.mjs + timeout: 90s + - type: prompt + name: Idiomatic Python quality + weight: 1 + config: + model: claude-sonnet-4.6 + prompt: |- + Evaluate whether the generated Python code is idiomatic. + Pass only when the code uses Pythonic naming, clear control flow, + explicit exception handling (no bare except), avoids wildcard imports, + and uses context managers or lifecycle-safe patterns where relevant. + Fail immediately if no Python code is generated. + Treat missing code as an automatic failure when there is no `.py` file in the workspace + and no substantive Python code in the assistant output. + scoring: scale_1_5 +constraints: + expect_skills: + - pydantic-models-py +environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/pydantic-models-py + files: + - src: ../../_shared/vally/tools/check-python-syntax.mjs + dest: .vally/tools/check-python-syntax.mjs + - src: ../../_shared/vally/tools/check-python-idiomatic.mjs + dest: .vally/tools/check-python-idiomatic.mjs + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json + - src: ./syntax-check-config.json + dest: .vally/syntax-check-config.json +artifacts: + include: + - "**/*.py" + - "**/requirements*.txt" + - "**/pyproject.toml" + - "**/setup.py" + - "**/README.md" + exclude: + - "**/.venv/**" + - "**/venv/**" + - "**/__pycache__/**" + - "**/*.pyc" + - "**/.mypy_cache/**" + - "**/.pytest_cache/**" diff --git a/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_experiment.yaml b/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_experiment.yaml new file mode 100644 index 00000000..9f89784e --- /dev/null +++ b/tests/scenarios/pydantic-models-py/vally/skill_effectiveness_experiment.yaml @@ -0,0 +1,20 @@ +# Purpose: Measure skill effectiveness by comparing the same stimulus with and without the corresponding skill. +name: pydantic-models-py-skill-experiment +evals: + - skill_effectiveness_eval.yaml +vary: + - /defaults/model + - /environment/skills +baseline: sonnet_baseline +variants: + sonnet_skill: + overrides: + model: claude-sonnet-4.6 + environment: + skills: + - ../../../../.github/plugins/azure-sdk-python/skills/pydantic-models-py + sonnet_baseline: + overrides: + model: claude-sonnet-4.6 + environment: + skills: [] diff --git a/tests/scenarios/pydantic-models-py/vally/syntax-check-config.json b/tests/scenarios/pydantic-models-py/vally/syntax-check-config.json new file mode 100644 index 00000000..4b941664 --- /dev/null +++ b/tests/scenarios/pydantic-models-py/vally/syntax-check-config.json @@ -0,0 +1,6 @@ +{ + "excludePatterns": [ + "*/assets/template.py", + "**/_**/template.py" + ] +} \ No newline at end of file