# Agent Operations Reference

## Agent Types and Kinds

```python
from azure.ai.projects.models import AgentKind

# Agent kinds
# - "prompt": Standard prompt-based agents
# - "hosted": Hosted agents
# - "container_app": Container App agents
# - "workflow": Workflow agents

# Filter agents by kind
agents = project_client.agents.list(kind=AgentKind.PROMPT)
```

## Basic Agent Creation

```python
agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="my-agent",
    instructions="You are a helpful assistant.",
)
print(f"Created agent, ID: {agent.id}")

# Clean up when done
project_client.agents.delete_agent(agent.id)
```

## Versioned Agents with PromptAgentDefinition

For production deployments, use versioned agents:

```python
from azure.ai.projects.models import PromptAgentDefinition

agent = project_client.agents.create_version(
    agent_name="customer-support-agent",
    definition=PromptAgentDefinition(
        model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
        instructions="You are a customer support specialist.",
        tools=[],  # Add tools as needed
    ),
    version_label="v1.0",
    description="Initial version",
)
print(f"Agent created (id: {agent.id}, name: {agent.name}, version: {agent.version})")
```

## Agent with Tools

```python
from azure.ai.agents.models import CodeInterpreterTool, FileSearchTool
from azure.ai.projects.models import PromptAgentDefinition

agent = project_client.agents.create_version(
    agent_name="tool-agent",
    definition=PromptAgentDefinition(
        model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
        instructions="You can execute code and search files.",
        tools=[CodeInterpreterTool(), FileSearchTool()],
    ),
)
```

## Agent with Response Format

### JSON Mode

```python
agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="json-agent",
    instructions="Always respond in JSON format.",
    response_format={"type": "json_object"},
)
```

### JSON Schema

```python
agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="schema-agent",
    instructions="Respond with weather data.",
    response_format={
        "type": "json_schema",
        "json_schema": {
            "name": "weather_response",
            "schema": {
                "type": "object",
                "properties": {
                    "temperature": {"type": "number"},
                    "conditions": {"type": "string"},
                    "humidity": {"type": "number"},
                },
                "required": ["temperature", "conditions"],
            },
        },
    },
)
```

## Thread Operations

### Create Thread

```python
thread = project_client.agents.threads.create()
print(f"Created thread, ID: {thread.id}")
```

### Create Thread with Tool Resources

```python
from azure.ai.agents.models import FileSearchTool

file_search = FileSearchTool(vector_store_ids=[vector_store.id])

thread = project_client.agents.threads.create(
    tool_resources=file_search.resources
)
```

### List Threads

```python
threads = project_client.agents.threads.list()
for thread in threads:
    print(f"Thread ID: {thread.id}")
```

## Message Operations

### Create Message

```python
message = project_client.agents.messages.create(
    thread_id=thread.id,
    role="user",
    content="What is the weather in Seattle?",
)
print(f"Created message, ID: {message.id}")
```

### Create Message with Attachment

```python
from azure.ai.agents.models import MessageAttachment, FileSearchTool

attachment = MessageAttachment(
    file_id=file.id,
    tools=FileSearchTool().definitions
)

message = project_client.agents.messages.create(
    thread_id=thread.id,
    role="user",
    content="What feature does Smart Eyewear offer?",
    attachments=[attachment],
)
```

### List Messages

```python
messages = project_client.agents.messages.list(thread_id=thread.id)
for msg in messages:
    print(f"Role: {msg.role}")
    for content in msg.content:
        if hasattr(content, 'text'):
            print(f"Content: {content.text.value}")
```

## Run Operations

### Create and Process Run

```python
run = project_client.agents.runs.create_and_process(
    thread_id=thread.id,
    agent_id=agent.id,
)
print(f"Run finished with status: {run.status}")

if run.status == "failed":
    print(f"Run failed: {run.last_error}")
```

### Create and Process with ToolSet

```python
from azure.ai.agents.models import FunctionTool, ToolSet

def get_weather(location: str) -> str:
    """Get weather for a location."""
    return f"Weather in {location}: 72F, sunny"

functions = FunctionTool(functions=[get_weather])
toolset = ToolSet()
toolset.add(functions)

# Enable auto function calls
project_client.agents.enable_auto_function_calls(toolset)

run = project_client.agents.runs.create_and_process(
    thread_id=thread.id,
    agent_id=agent.id,
    toolset=toolset,  # Pass toolset for auto-execution
)
```

### Streaming Run

```python
from azure.ai.agents.models import AgentEventHandler

class MyHandler(AgentEventHandler):
    def on_message_delta(self, delta):
        if delta.text:
            print(delta.text.value, end="", flush=True)

    def on_error(self, data):
        print(f"Error: {data}")

with project_client.agents.runs.stream(
    thread_id=thread.id,
    agent_id=agent.id,
    event_handler=MyHandler(),
) as stream:
    stream.until_done()
```

## File Operations

### Upload File

```python
from azure.ai.agents.models import FilePurpose

file = project_client.agents.files.upload_and_poll(
    file_path="./data/document.pdf",
    purpose=FilePurpose.AGENTS,
)
print(f"Uploaded file, ID: {file.id}")
```

### Create Vector Store

```python
vector_store = project_client.agents.vector_stores.create_and_poll(
    file_ids=[file.id],
    name="my-vector-store",
)
print(f"Created vector store, ID: {vector_store.id}")
```

## Agent Lifecycle Best Practices

```python
# 1. Use context managers
with project_client:
    agent = project_client.agents.create_agent(...)
    thread = project_client.agents.threads.create()
    
    # ... use agent
    
    # Clean up
    project_client.agents.delete_agent(agent.id)

# 2. For versioned agents, manage versions explicitly
agent_v1 = project_client.agents.create_version(
    agent_name="my-agent",
    definition=PromptAgentDefinition(...),
    version_label="v1.0",
)

agent_v2 = project_client.agents.create_version(
    agent_name="my-agent",
    definition=PromptAgentDefinition(...),
    version_label="v2.0",
)

# 3. Reuse threads for conversation continuity
thread_id = thread.id  # Save for later

# Resume conversation
project_client.agents.messages.create(
    thread_id=thread_id,
    role="user",
    content="Follow-up question",
)
```

# Azure AI Projects SDK - Complete API Reference

**Package**: `azure-ai-projects` v2.0.0b4  
**Repository**: [Azure/azure-sdk-for-python](https://github.com/Azure/azure-sdk-for-python)  
**Path**: `sdk/ai/azure-ai-projects/`  
**Commit**: `7e86ab0076297173aae290c11fa14660bed2b125`

---

## Table of Contents

1. [Client Classes](#1-client-classes)
2. [Agent Classes](#2-agent-classes)
3. [Tool Classes](#3-tool-classes)
4. [ItemResource Classes](#4-itemresource-classes)
5. [InputItem Classes](#5-inputitem-classes)
6. [Index Classes](#6-index-classes)
7. [Evaluation Classes](#7-evaluation-classes)
8. [Memory Classes](#8-memory-classes)
9. [Schedule & Trigger Classes](#9-schedule--trigger-classes)
10. [Credential Classes](#10-credential-classes)
11. [ComputerAction Classes](#11-computeraction-classes)
12. [WebSearch Classes](#12-websearch-classes)
13. [Insight Classes](#13-insight-classes)
14. [Filter Classes](#14-filter-classes)
15. [Annotation Classes](#15-annotation-classes)
16. [Response & Output Classes](#16-response--output-classes)
17. [All Enums](#17-all-enums)

---

## 1. Client Classes

### AIProjectClient (Sync)

```python
from azure.ai.projects import AIProjectClient
from azure.identity import DefaultAzureCredential

client = AIProjectClient(
    endpoint="https://<resource>.services.ai.azure.com/api/projects/<project>",
    credential=DefaultAzureCredential(),
)

# Operations
client.agents          # AgentsOperations
client.connections     # ConnectionsOperations
client.deployments     # DeploymentsOperations
client.datasets        # DatasetsOperations
client.indexes         # IndexesOperations
client.evaluations     # EvaluationsOperations
client.red_teams       # RedTeamsOperations
client.get_openai_client()  # Returns OpenAI-compatible client
```

### AIProjectClient (Async)

```python
from azure.ai.projects.aio import AIProjectClient
from azure.identity.aio import DefaultAzureCredential

async with AIProjectClient(
    endpoint="https://<resource>.services.ai.azure.com/api/projects/<project>",
    credential=DefaultAzureCredential(),
) as client:
    agent = await client.agents.create_agent(...)
```

---

## 2. Agent Classes

### Agent Definition Hierarchy

```
AgentDefinition (base)
├── PromptAgentDefinition         # AgentKind.PROMPT
├── HostedAgentDefinition         # AgentKind.HOSTED
├── ImageBasedHostedAgentDefinition  # AgentKind.IMAGE_BASED_HOSTED
├── ContainerAppAgentDefinition   # AgentKind.CONTAINER_APP
└── WorkflowAgentDefinition       # AgentKind.WORKFLOW
```

### PromptAgentDefinition

```python
from azure.ai.projects.models import PromptAgentDefinition

definition = PromptAgentDefinition(
    model="gpt-4o-mini",                           # Required
    instructions="You are helpful.",               # Optional
    name="my-agent",                               # Optional
    description="Agent description",               # Optional
    tools=[CodeInterpreterTool()],                 # Optional[List[Tool]]
    tool_resources={...},                          # Optional[Dict]
    response_format={"type": "json_object"},       # Optional
    temperature=0.7,                               # Optional[float]
    top_p=1.0,                                     # Optional[float]
    metadata={"key": "value"},                     # Optional[Dict]
)
```

### HostedAgentDefinition

```python
from azure.ai.projects.models import HostedAgentDefinition

definition = HostedAgentDefinition(
    model="gpt-4o-mini",
    instructions="You are helpful.",
    protocol_version="2025-05-01",  # Hosted agent protocol version
)
```

### ImageBasedHostedAgentDefinition

```python
from azure.ai.projects.models import ImageBasedHostedAgentDefinition

definition = ImageBasedHostedAgentDefinition(
    image_id="acr.azurecr.io/my-agent:latest",
    environment_variables={"KEY": "value"},
    protocol_version="2025-05-01",
)
```

### ContainerAppAgentDefinition

```python
from azure.ai.projects.models import ContainerAppAgentDefinition

definition = ContainerAppAgentDefinition(
    container_app_id="/subscriptions/.../containerApps/my-app",
)
```

### WorkflowAgentDefinition

```python
from azure.ai.projects.models import WorkflowAgentDefinition

definition = WorkflowAgentDefinition(
    csdl="<workflow CSDL definition>",  # Common Schema Definition Language
)
```

### AgentKind Enum

```python
from azure.ai.projects.models import AgentKind

AgentKind.PROMPT              # "prompt"
AgentKind.HOSTED              # "hosted"
AgentKind.IMAGE_BASED_HOSTED  # "image_based_hosted"
AgentKind.CONTAINER_APP       # "container_app"
AgentKind.WORKFLOW            # "workflow"
```

---

## 3. Tool Classes

### Tool Type Hierarchy

```
Tool (base)
├── CodeInterpreterTool
├── FileSearchTool
├── FunctionTool
├── AzureAISearchTool
├── AzureFunctionTool
├── BingGroundingTool
├── BingCustomSearchPreviewTool
├── OpenApiTool
├── MCPTool
├── ImageGenTool
├── ComputerUsePreviewTool
├── WebSearchTool
├── WebSearchPreviewTool
├── A2APreviewTool
├── BrowserAutomationPreviewTool
├── CaptureStructuredOutputsTool
├── MicrosoftFabricPreviewTool
├── SharepointPreviewTool
├── MemorySearchPreviewTool
├── LocalShellToolParam
├── FunctionShellToolParam
├── ApplyPatchToolParam
└── CustomToolParam
```

### CodeInterpreterTool

```python
from azure.ai.projects.models import CodeInterpreterTool

tool = CodeInterpreterTool(
    container=CodeInterpreterContainerAuto(),  # Optional: auto-managed container
)
```

### FileSearchTool

```python
from azure.ai.projects.models import FileSearchTool

tool = FileSearchTool(
    vector_store_ids=["vs_abc123"],           # Optional[List[str]]
    ranking_options=RankingOptions(...),       # Optional
    max_num_results=10,                        # Optional[int]
)
```

### FunctionTool

```python
from azure.ai.projects.models import FunctionTool

def get_weather(location: str) -> str:
    """Get weather for a location."""
    return f"Weather in {location}: 72F"

tool = FunctionTool(functions=[get_weather])
```

### AzureAISearchTool

```python
from azure.ai.projects.models import (
    AzureAISearchTool,
    AzureAISearchToolResource,
    AISearchIndexResource,
)

tool = AzureAISearchTool(
    resources=[
        AzureAISearchToolResource(
            index=AISearchIndexResource(
                index_connection_id="conn_id",
                index_name="my-index",
            ),
            query_type=AzureAISearchQueryType.VECTOR,
        )
    ]
)
```

### AzureFunctionTool

```python
from azure.ai.projects.models import (
    AzureFunctionTool,
    AzureFunctionDefinition,
    AzureFunctionBinding,
    AzureFunctionStorageQueue,
)

tool = AzureFunctionTool(
    function=AzureFunctionDefinition(
        function=AzureFunctionDefinitionFunction(
            name="my-function",
            description="Processes data",
            parameters={
                "type": "object",
                "properties": {"input": {"type": "string"}},
            },
        ),
        input_binding=AzureFunctionBinding(
            storage_queue=AzureFunctionStorageQueue(
                queue_service_uri="https://...",
                queue_name="input-queue",
            )
        ),
        output_binding=AzureFunctionBinding(...),
    )
)
```

### BingGroundingTool

```python
from azure.ai.projects.models import (
    BingGroundingTool,
    BingGroundingSearchConfiguration,
    BingGroundingSearchToolParameters,
)

tool = BingGroundingTool(
    bing_grounding=BingGroundingSearchToolParameters(
        connection_id="bing_connection_id",
        configuration=BingGroundingSearchConfiguration(
            market="en-US",
            set_lang="en",
        ),
    )
)
```

### OpenApiTool

```python
from azure.ai.projects.models import (
    OpenApiTool,
    OpenApiFunctionDefinition,
    OpenApiAnonymousAuthDetails,
)

tool = OpenApiTool(
    openapi=OpenApiFunctionDefinition(
        name="weather-api",
        description="Get weather data",
        spec={"openapi": "3.0.0", ...},
        auth=OpenApiAnonymousAuthDetails(),
    )
)
```

### MCPTool

```python
from azure.ai.projects.models import (
    MCPTool,
    MCPToolFilter,
    MCPToolRequireApproval,
)

tool = MCPTool(
    server_label="my-mcp-server",
    server_url="http://localhost:8000/mcp",
    allowed_tools=MCPToolFilter(tool_names=["search", "fetch"]),
    require_approval=MCPToolRequireApproval.ALWAYS,
)
```

### ComputerUsePreviewTool

```python
from azure.ai.projects.models import ComputerUsePreviewTool, ComputerEnvironment

tool = ComputerUsePreviewTool(
    display_width=1920,
    display_height=1080,
    environment=ComputerEnvironment.BROWSER,
)
```

### ToolType Enum (23 values)

```python
from azure.ai.projects.models import ToolType

ToolType.CODE_INTERPRETER           # "code_interpreter"
ToolType.FILE_SEARCH                # "file_search"
ToolType.FUNCTION                   # "function"
ToolType.BING_GROUNDING             # "bing_grounding"
ToolType.AZURE_AI_SEARCH            # "azure_ai_search"
ToolType.AZURE_FUNCTION             # "azure_function"
ToolType.OPENAPI                    # "openapi"
ToolType.MCP                        # "mcp"
ToolType.IMAGE_GEN                  # "image_gen"
ToolType.COMPUTER_USE_PREVIEW       # "computer_use_preview"
ToolType.WEB_SEARCH                 # "web_search"
ToolType.WEB_SEARCH_PREVIEW         # "web_search_preview"
ToolType.A2A_PREVIEW                # "a2a_preview"
ToolType.BROWSER_AUTOMATION_PREVIEW # "browser_automation_preview"
ToolType.CAPTURE_STRUCTURED_OUTPUTS # "capture_structured_outputs"
ToolType.MICROSOFT_FABRIC_PREVIEW   # "microsoft_fabric_preview"
ToolType.SHAREPOINT_PREVIEW         # "sharepoint_preview"
ToolType.MEMORY_SEARCH_PREVIEW      # "memory_search_preview"
ToolType.BING_CUSTOM_SEARCH_PREVIEW # "bing_custom_search_preview"
ToolType.LOCAL_SHELL                # "local_shell"
ToolType.FUNCTION_SHELL             # "function_shell"
ToolType.APPLY_PATCH                # "apply_patch"
ToolType.CUSTOM                     # "custom"
```

---

## 4. ItemResource Classes

Output item types returned from agent runs:

| Class | ItemResourceType | Description |
|-------|------------------|-------------|
| `ItemResourceFunctionToolCallResource` | `FUNCTION_CALL` | Function call result |
| `ItemResourceCodeInterpreterToolCall` | `CODE_INTERPRETER_CALL` | Code execution |
| `ItemResourceFileSearchToolCall` | `FILE_SEARCH_CALL` | File search result |
| `ItemResourceMcpToolCall` | `MCP_CALL` | MCP tool call |
| `ItemResourceComputerToolCall` | `COMPUTER_CALL` | Computer action |
| `ItemResourceWebSearchToolCall` | `WEB_SEARCH_CALL` | Web search result |
| `ItemResourceImageGenToolCall` | `IMAGE_GEN_CALL` | Image generation |
| `ItemResourceApplyPatchToolCall` | `APPLY_PATCH_CALL` | Patch application |
| `ItemResourceLocalShellToolCall` | `LOCAL_SHELL_CALL` | Shell command |
| `ItemResourceFunctionShellCall` | `FUNCTION_SHELL_CALL` | Function shell |
| `ItemResourceOutputMessage` | `MESSAGE` | Output message |
| `StructuredOutputsItemResource` | `STRUCTURED_OUTPUTS` | Structured output |
| `WorkflowActionOutputItemResource` | `WORKFLOW_ACTION_OUTPUT` | Workflow output |
| `OAuthConsentRequestItemResource` | `OAUTH_CONSENT_REQUEST` | OAuth consent |
| `ItemResourceMcpApprovalRequest` | `MCP_APPROVAL_REQUEST` | MCP approval |
| `ItemResourceMcpListTools` | `MCP_LIST_TOOLS` | MCP tools list |

---

## 5. InputItem Classes

Input item types for agent interactions:

| Class | InputItemType | Description |
|-------|---------------|-------------|
| `InputItemFunctionToolCall` | `FUNCTION_CALL` | Function call input |
| `InputItemCodeInterpreterToolCall` | `CODE_INTERPRETER_CALL` | Code input |
| `InputItemFileSearchToolCall` | `FILE_SEARCH_CALL` | Search input |
| `InputItemMcpToolCall` | `MCP_CALL` | MCP input |
| `InputItemComputerToolCall` | `COMPUTER_CALL` | Computer input |
| `InputItemWebSearchToolCall` | `WEB_SEARCH_CALL` | Web search input |
| `InputItemOutputMessage` | `MESSAGE` | Message input |
| `InputItemReasoningItem` | `REASONING` | Reasoning input |
| `InputItemMcpApprovalRequest` | `MCP_APPROVAL_REQUEST` | Approval input |
| `InputItemMcpApprovalResponse` | `MCP_APPROVAL_RESPONSE` | Approval response |
| `InputItemCustomToolCall` | `CUSTOM_TOOL_CALL` | Custom tool input |
| `InputItemCustomToolCallOutput` | `CUSTOM_TOOL_CALL_OUTPUT` | Custom output |

---

## 6. Index Classes

```python
from azure.ai.projects.models import (
    Index,
    AISearchIndexResource,
    ManagedAzureAISearchIndex,
    CosmosDBIndex,
    AzureAISearchIndex,
    IndexType,
)

# Azure AI Search Index
index = AzureAISearchIndex(
    connection_id="search_connection_id",
    index_name="my-index",
)

# Managed Azure AI Search Index
managed_index = ManagedAzureAISearchIndex(
    embedding_model_connection="embedding_conn",
    embedding_model_deployment="text-embedding-ada-002",
)

# Cosmos DB Index
cosmos_index = CosmosDBIndex(
    connection_id="cosmos_connection_id",
    database_name="my-db",
    container_name="my-container",
)
```

### IndexType Enum

```python
IndexType.AZURE_AI_SEARCH          # "azure_ai_search"
IndexType.MANAGED_AZURE_AI_SEARCH  # "managed_azure_ai_search"
IndexType.COSMOS_DB                # "cosmos_db"
```

---

## 7. Evaluation Classes

```python
from azure.ai.projects.models import (
    # Core evaluation
    Evaluator,
    EvaluatorDefinition,
    EvaluatorVersion,
    EvaluatorMetric,
    EvalResult,
    
    # Rules and actions
    EvaluationRule,
    EvaluationRuleAction,
    ContinuousEvaluationRuleAction,
    HumanEvaluationRuleAction,
    
    # Comparison reports
    EvalCompareReport,
    EvalRunResultComparison,
    EvalRunResultSummary,
    EvalRunResultCompareItem,
    EvaluationResultSample,
    
    # Evaluator definitions
    CodeBasedEvaluatorDefinition,
    PromptBasedEvaluatorDefinition,
    
    # Taxonomy
    EvaluationTaxonomy,
    EvaluationTaxonomyInput,
    TaxonomyCategory,
    TaxonomySubCategory,
)
```

### EvaluatorType Enum

```python
from azure.ai.projects.models import EvaluatorType

EvaluatorType.GROUNDEDNESS
EvaluatorType.RELEVANCE
EvaluatorType.COHERENCE
EvaluatorType.FLUENCY
EvaluatorType.SIMILARITY
EvaluatorType.F1_SCORE
EvaluatorType.RETRIEVAL_SCORE
EvaluatorType.HATE_UNFAIRNESS
EvaluatorType.VIOLENCE
EvaluatorType.SELF_HARM
EvaluatorType.SEXUAL
EvaluatorType.PROTECTED_MATERIAL_TEXT
EvaluatorType.INDIRECT_ATTACK
EvaluatorType.CODE_VULNERABILITY
EvaluatorType.CUSTOM
```

---

## 8. Memory Classes

```python
from azure.ai.projects.models import (
    # Memory items
    MemoryItem,
    ChatSummaryMemoryItem,
    UserProfileMemoryItem,
    
    # Memory stores
    MemoryStoreDefinition,
    MemoryStoreDefaultDefinition,
    MemoryStoreDefaultOptions,
    MemoryStoreDetails,
    
    # Memory operations
    MemoryOperation,
    MemorySearchItem,
    MemorySearchOptions,
    MemorySearchPreviewTool,
    
    # Results
    MemoryStoreSearchResult,
    MemoryStoreUpdateResult,
    MemoryStoreDeleteScopeResult,
)
```

### MemoryItemKind Enum

```python
from azure.ai.projects.models import MemoryItemKind

MemoryItemKind.CHAT_SUMMARY    # "chat_summary"
MemoryItemKind.USER_PROFILE    # "user_profile"
```

### MemoryStoreKind Enum

```python
from azure.ai.projects.models import MemoryStoreKind

MemoryStoreKind.DEFAULT        # "default"
```

### MemoryOperationKind Enum

```python
from azure.ai.projects.models import MemoryOperationKind

MemoryOperationKind.ADD        # "add"
MemoryOperationKind.REMOVE     # "remove"
```

---

## 9. Schedule & Trigger Classes

### Schedule Classes

```python
from azure.ai.projects.models import (
    Schedule,
    ScheduleRun,
    ScheduleTask,
    EvaluationScheduleTask,
    InsightScheduleTask,
)

# Create a schedule
schedule = Schedule(
    name="daily-eval",
    trigger=CronTrigger(expression="0 0 * * *"),
    task=EvaluationScheduleTask(
        evaluation_id="eval_123",
    ),
)
```

### Trigger Classes

```python
from azure.ai.projects.models import (
    Trigger,
    CronTrigger,
    RecurrenceTrigger,
    OneTimeTrigger,
)

# Cron trigger
cron = CronTrigger(
    expression="0 9 * * MON-FRI",  # 9 AM weekdays
    time_zone="America/Los_Angeles",
)

# Recurrence trigger
recurrence = RecurrenceTrigger(
    frequency=RecurrenceType.DAILY,
    interval=1,
    schedule=DailyRecurrenceSchedule(hours=[9, 17], minutes=[0]),
)

# One-time trigger
one_time = OneTimeTrigger(
    run_at="2026-02-01T09:00:00Z",
)
```

### RecurrenceSchedule Classes

```python
from azure.ai.projects.models import (
    RecurrenceSchedule,
    HourlyRecurrenceSchedule,
    DailyRecurrenceSchedule,
    WeeklyRecurrenceSchedule,
    MonthlyRecurrenceSchedule,
    DayOfWeek,
)

# Hourly
hourly = HourlyRecurrenceSchedule(minutes=[0, 30])

# Daily
daily = DailyRecurrenceSchedule(hours=[9, 17], minutes=[0])

# Weekly
weekly = WeeklyRecurrenceSchedule(
    days=[DayOfWeek.MONDAY, DayOfWeek.WEDNESDAY, DayOfWeek.FRIDAY],
    hours=[9],
    minutes=[0],
)

# Monthly
monthly = MonthlyRecurrenceSchedule(
    month_days=[1, 15],
    hours=[9],
    minutes=[0],
)
```

### TriggerType Enum

```python
from azure.ai.projects.models import TriggerType

TriggerType.CRON        # "cron"
TriggerType.RECURRENCE  # "recurrence"
TriggerType.ONE_TIME    # "one_time"
```

### RecurrenceType Enum

```python
from azure.ai.projects.models import RecurrenceType

RecurrenceType.HOURLY   # "hourly"
RecurrenceType.DAILY    # "daily"
RecurrenceType.WEEKLY   # "weekly"
RecurrenceType.MONTHLY  # "monthly"
```

---

## 10. Credential Classes

```python
from azure.ai.projects.models import (
    BaseCredentials,
    ApiKeyCredentials,
    EntraIDCredentials,
    SASCredentials,
    NoAuthenticationCredentials,
    AgenticIdentityCredentials,
    CustomCredential,
    CredentialType,
)

# API Key
api_key_cred = ApiKeyCredentials(key="sk-...")

# Entra ID (Azure AD)
entra_cred = EntraIDCredentials(
    client_id="...",
    client_secret="...",
    tenant_id="...",
)

# SAS Token
sas_cred = SASCredentials(sas_token="?sv=...")

# No authentication
no_auth = NoAuthenticationCredentials()

# Agentic Identity
agentic_cred = AgenticIdentityCredentials()
```

### CredentialType Enum

```python
from azure.ai.projects.models import CredentialType

CredentialType.API_KEY              # "api_key"
CredentialType.ENTRA_ID             # "entra_id"
CredentialType.SAS                  # "sas"
CredentialType.NO_AUTH              # "no_auth"
CredentialType.AGENTIC_IDENTITY     # "agentic_identity"
CredentialType.CUSTOM               # "custom"
```

---

## 11. ComputerAction Classes

```python
from azure.ai.projects.models import (
    ComputerAction,
    ClickParam,
    DoubleClickAction,
    Drag,
    DragPoint,
    KeyPressAction,
    Move,
    Screenshot,
    Scroll,
    Type,
    Wait,
    ComputerActionType,
)

# Click
click = ClickParam(
    x=100,
    y=200,
    button=ClickButtonType.LEFT,
)

# Double click
dbl_click = DoubleClickAction(x=100, y=200)

# Drag
drag = Drag(
    path=[DragPoint(x=0, y=0), DragPoint(x=100, y=100)],
)

# Key press
key = KeyPressAction(keys=["ctrl", "c"])

# Move
move = Move(x=300, y=400)

# Screenshot
screenshot = Screenshot()

# Scroll
scroll = Scroll(x=0, y=500, delta_x=0, delta_y=-100)

# Type text
type_text = Type(text="Hello, World!")

# Wait
wait = Wait(ms=1000)
```

### ComputerActionType Enum

```python
from azure.ai.projects.models import ComputerActionType

ComputerActionType.CLICK          # "click"
ComputerActionType.DOUBLE_CLICK   # "double_click"
ComputerActionType.DRAG           # "drag"
ComputerActionType.KEY_PRESS      # "keypress"
ComputerActionType.MOVE           # "move"
ComputerActionType.SCREENSHOT     # "screenshot"
ComputerActionType.SCROLL         # "scroll"
ComputerActionType.TYPE           # "type"
ComputerActionType.WAIT           # "wait"
```

---

## 12. WebSearch Classes

```python
from azure.ai.projects.models import (
    WebSearchTool,
    WebSearchPreviewTool,
    WebSearchToolFilters,
    WebSearchApproximateLocation,
    WebSearchConfiguration,
    SearchContextSize,
)

# Web Search Tool
tool = WebSearchTool(
    user_location=WebSearchApproximateLocation(
        city="Seattle",
        region="Washington",
        country="US",
    ),
    search_context_size=SearchContextSize.MEDIUM,
)

# Web Search Preview Tool
preview_tool = WebSearchPreviewTool(
    configuration=WebSearchConfiguration(
        filters=WebSearchToolFilters(
            domains=["docs.microsoft.com", "learn.microsoft.com"],
        ),
    ),
)
```

### Web Search Action Classes

```python
from azure.ai.projects.models import (
    WebSearchActionFind,
    WebSearchActionOpenPage,
    WebSearchActionSearch,
    WebSearchActionSearchSources,
)
```

### SearchContextSize Enum

```python
from azure.ai.projects.models import SearchContextSize

SearchContextSize.LOW     # "low"
SearchContextSize.MEDIUM  # "medium"
SearchContextSize.HIGH    # "high"
```

---

## 13. Insight Classes

```python
from azure.ai.projects.models import (
    # Core
    Insight,
    InsightRequest,
    InsightResult,
    InsightCluster,
    InsightSummary,
    InsightsMetadata,
    InsightSample,
    
    # Model configuration
    InsightModelConfiguration,
    
    # Specialized results
    ClusterInsightResult,
    ClusterTokenUsage,
    AgentClusterInsightResult,
    AgentClusterInsightsRequest,
    EvaluationRunClusterInsightResult,
    EvaluationRunClusterInsightsRequest,
)
```

### InsightType Enum

```python
from azure.ai.projects.models import InsightType

InsightType.CLUSTER  # "cluster"
```

---

## 14. Filter Classes

```python
from azure.ai.projects.models import (
    ComparisonFilter,
    CompoundFilter,
    MCPToolFilter,
    MCPToolRequireApproval,
)

# Comparison filter
comparison = ComparisonFilter(
    field="category",
    operator="eq",
    value="documentation",
)

# Compound filter
compound = CompoundFilter(
    logical_operator="and",
    filters=[comparison1, comparison2],
)

# MCP tool filter
mcp_filter = MCPToolFilter(
    tool_names=["search", "fetch", "analyze"],
)
```

---

## 15. Annotation Classes

```python
from azure.ai.projects.models import (
    Annotation,
    FileCitationBody,
    ContainerFileCitationBody,
    UrlCitationBody,
    AnnotationType,
)
```

### AnnotationType Enum

```python
from azure.ai.projects.models import AnnotationType

AnnotationType.FILE_CITATION           # "file_citation"
AnnotationType.CONTAINER_FILE_CITATION # "container_file_citation"
AnnotationType.URL_CITATION            # "url_citation"
AnnotationType.FILE_PATH               # "file_path"
```

---

## 16. Response & Output Classes

```python
from azure.ai.projects.models import (
    # Output content
    OutputContent,
    OutputMessageContent,
    OutputMessageContentOutputTextContent,
    OutputMessageContentRefusalContent,
    
    # Usage details
    ResponseUsageInputTokensDetails,
    ResponseUsageOutputTokensDetails,
    
    # Delete responses
    DeleteAgentResponse,
    DeleteAgentVersionResponse,
    DeleteMemoryStoreResult,
)
```

---

## 17. All Enums (67 Total)

### Agent & Protocol Enums
```python
AgentKind              # PROMPT, HOSTED, IMAGE_BASED_HOSTED, CONTAINER_APP, WORKFLOW
AgentProtocol          # Protocol versions
```

### Tool Enums
```python
ToolType               # 23 tool types
ComputerActionType     # CLICK, DOUBLE_CLICK, DRAG, KEY_PRESS, MOVE, SCREENSHOT, SCROLL, TYPE, WAIT
ClickButtonType        # LEFT, RIGHT, MIDDLE, WHEEL
ComputerEnvironment    # BROWSER, DESKTOP, ...
OpenApiAuthType        # ANONYMOUS, MANAGED, PROJECT_CONNECTION
MCPToolCallStatus      # PENDING, APPROVED, REJECTED
```

### Item Type Enums
```python
ItemResourceType       # 26 output item types
InputItemType          # 20+ input item types
InputContentType       # TEXT, IMAGE, FILE
OutputContentType      # TEXT, REFUSAL
OutputMessageContentType  # OUTPUT_TEXT, REFUSAL
```

### Index & Data Enums
```python
IndexType              # AZURE_AI_SEARCH, MANAGED_AZURE_AI_SEARCH, COSMOS_DB
DatasetType            # FILE, FOLDER
AzureAISearchQueryType # SIMPLE, FULL, SEMANTIC, VECTOR, VECTOR_SIMPLE_HYBRID, ...
```

### Evaluation Enums
```python
EvaluatorType          # GROUNDEDNESS, RELEVANCE, COHERENCE, FLUENCY, etc.
EvaluatorCategory      # QUALITY, SAFETY, CUSTOM
EvaluatorDefinitionType # CODE_BASED, PROMPT_BASED
EvaluatorMetricType    # NUMERIC, BOOLEAN, STRING
EvaluatorMetricDirection # HIGHER_IS_BETTER, LOWER_IS_BETTER
RiskCategory           # HATE_UNFAIRNESS, VIOLENCE, SELF_HARM, SEXUAL, etc.
SampleType             # GOOD, BAD, NEUTRAL
```

### Memory Enums
```python
MemoryItemKind         # CHAT_SUMMARY, USER_PROFILE
MemoryStoreKind        # DEFAULT
MemoryOperationKind    # ADD, REMOVE
MemoryStoreUpdateStatus # COMPLETED, PENDING, FAILED
```

### Schedule & Trigger Enums
```python
TriggerType            # CRON, RECURRENCE, ONE_TIME
RecurrenceType         # HOURLY, DAILY, WEEKLY, MONTHLY
DayOfWeek              # MONDAY through SUNDAY
ScheduleTaskType       # EVALUATION, INSIGHT
ScheduleProvisioningStatus # PROVISIONING, PROVISIONED, FAILED
```

### Credential Enums
```python
CredentialType         # API_KEY, ENTRA_ID, SAS, NO_AUTH, AGENTIC_IDENTITY, CUSTOM
```

### Connection Enums
```python
ConnectionType         # AZURE_OPEN_AI, AZURE_AI_SEARCH, AZURE_BLOB, etc.
DeploymentType         # MODEL, EMBEDDING, etc.
```

### Annotation Enums
```python
AnnotationType         # FILE_CITATION, CONTAINER_FILE_CITATION, URL_CITATION, FILE_PATH
```

### Status Enums
```python
OperationState         # PENDING, IN_PROGRESS, COMPLETED, FAILED
FunctionCallItemStatus # IN_PROGRESS, COMPLETED, INCOMPLETE
LocalShellCallStatus   # PENDING, COMPLETED, ERROR
ApplyPatchCallStatus   # PENDING, COMPLETED, FAILED
```

### Miscellaneous Enums
```python
PageOrder              # ASC, DESC
ImageDetail            # LOW, HIGH, AUTO
InputFidelity          # LOW, MEDIUM, HIGH
SearchContextSize      # LOW, MEDIUM, HIGH
RankerVersionType      # V1, V2
GrammarSyntax1         # BNF, EBNF
TextResponseFormatConfigurationType  # TEXT, JSON_OBJECT, JSON_SCHEMA
ContainerLogKind       # STDOUT, STDERR
ContainerMemoryLimit   # 256MB, 512MB, 1GB, etc.
DetailEnum             # LOW, HIGH, AUTO
AttackStrategy         # Various attack strategies for red teaming
TreatmentEffectType    # POSITIVE, NEGATIVE, NEUTRAL
```

---

## Quick Reference: Common Import Patterns

### Client Initialization
```python
from azure.ai.projects import AIProjectClient
from azure.identity import DefaultAzureCredential
import os

client = AIProjectClient(
    endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
    credential=DefaultAzureCredential(),
)
```

### Agent with Tools
```python
from azure.ai.projects.models import (
    PromptAgentDefinition,
    CodeInterpreterTool,
    FileSearchTool,
    FunctionTool,
    AzureAISearchTool,
    BingGroundingTool,
)
```

### All Tool Imports
```python
from azure.ai.projects.models import (
    # Core tools
    CodeInterpreterTool,
    FileSearchTool,
    FunctionTool,
    # Azure tools
    AzureAISearchTool,
    AzureFunctionTool,
    BingGroundingTool,
    BingCustomSearchPreviewTool,
    # External tools
    OpenApiTool,
    MCPTool,
    # Computer use
    ComputerUsePreviewTool,
    # Web & preview
    WebSearchTool,
    WebSearchPreviewTool,
    A2APreviewTool,
    BrowserAutomationPreviewTool,
    # Enterprise
    MicrosoftFabricPreviewTool,
    SharepointPreviewTool,
    # Memory
    MemorySearchPreviewTool,
    # Utility
    ImageGenTool,
    CaptureStructuredOutputsTool,
    LocalShellToolParam,
    FunctionShellToolParam,
    ApplyPatchToolParam,
    CustomToolParam,
)
```

---

*Document generated from azure-ai-projects v2.0.0b4 source code analysis.*
*Last updated: February 2026*

# Async Patterns Reference

## Async Client Setup

```python
import os
import asyncio
from azure.ai.projects.aio import AIProjectClient
from azure.identity.aio import DefaultAzureCredential

# Requires: pip install aiohttp

async def main():
    async with (
        DefaultAzureCredential() as credential,
        AIProjectClient(
            endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
            credential=credential,
        ) as client,
    ):
        # Use async operations
        pass

asyncio.run(main())
```

## Async Agent Operations

### Create Agent

```python
async with AIProjectClient(
    endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
    credential=DefaultAzureCredential(),
) as client:
    agent = await client.agents.create_agent(
        model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
        name="async-agent",
        instructions="You are helpful.",
    )
    print(f"Created agent: {agent.id}")
    
    # Clean up
    await client.agents.delete_agent(agent.id)
```

### Full Conversation Flow

```python
import os
import asyncio
from azure.ai.projects.aio import AIProjectClient
from azure.identity.aio import DefaultAzureCredential

async def async_conversation():
    async with (
        DefaultAzureCredential() as credential,
        AIProjectClient(
            endpoint=os.environ["AZURE_AI_PROJECT_ENDPOINT"],
            credential=credential,
        ) as client,
    ):
        # Create agent
        agent = await client.agents.create_agent(
            model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
            name="async-agent",
            instructions="You are a helpful assistant.",
        )
        
        # Create thread
        thread = await client.agents.threads.create()
        
        # Add message
        await client.agents.messages.create(
            thread_id=thread.id,
            role="user",
            content="What is the capital of Japan?",
        )
        
        # Create and process run
        run = await client.agents.runs.create_and_process(
            thread_id=thread.id,
            agent_id=agent.id,
        )
        
        # Get messages
        if run.status == "completed":
            messages = await client.agents.messages.list(thread_id=thread.id)
            async for msg in messages:
                if msg.role == "assistant":
                    print(f"Response: {msg.content[0].text.value}")
        
        # Clean up
        await client.agents.delete_agent(agent.id)

asyncio.run(async_conversation())
```

## Async File Operations

```python
from azure.ai.agents.models import FilePurpose

async with AIProjectClient(...) as client:
    # Upload file
    file = await client.agents.files.upload_and_poll(
        file_path="./data/document.pdf",
        purpose=FilePurpose.AGENTS,
    )
    
    # Create vector store
    vector_store = await client.agents.vector_stores.create_and_poll(
        file_ids=[file.id],
        name="async-vector-store",
    )
```

## Async Connections and Deployments

```python
async with AIProjectClient(...) as client:
    # List connections
    connections = client.connections.list()
    async for conn in connections:
        print(f"Connection: {conn.name}")
    
    # List deployments
    deployments = client.deployments.list()
    async for deployment in deployments:
        print(f"Deployment: {deployment.name}")
```

## Async Streaming

```python
from azure.ai.agents.aio import AsyncAgentEventHandler

class AsyncHandler(AsyncAgentEventHandler):
    async def on_message_delta(self, delta):
        if delta.text:
            print(delta.text.value, end="", flush=True)
    
    async def on_error(self, data):
        print(f"Error: {data}")

async with AIProjectClient(...) as client:
    async with client.agents.runs.stream(
        thread_id=thread.id,
        agent_id=agent.id,
        event_handler=AsyncHandler(),
    ) as stream:
        await stream.until_done()
```

## Concurrent Operations

```python
import asyncio

async def process_multiple_queries(client, agent_id, queries):
    """Process multiple queries concurrently."""
    
    async def process_query(query):
        thread = await client.agents.threads.create()
        await client.agents.messages.create(
            thread_id=thread.id,
            role="user",
            content=query,
        )
        run = await client.agents.runs.create_and_process(
            thread_id=thread.id,
            agent_id=agent_id,
        )
        if run.status == "completed":
            messages = await client.agents.messages.list(thread_id=thread.id)
            async for msg in messages:
                if msg.role == "assistant":
                    return msg.content[0].text.value
        return None
    
    # Process all queries concurrently
    results = await asyncio.gather(*[process_query(q) for q in queries])
    return results

# Usage
async with AIProjectClient(...) as client:
    queries = [
        "What is Python?",
        "What is JavaScript?",
        "What is Rust?",
    ]
    results = await process_multiple_queries(client, agent.id, queries)
    for query, result in zip(queries, results):
        print(f"Q: {query}")
        print(f"A: {result}\n")
```

## Error Handling

```python
from azure.core.exceptions import HttpResponseError

async with AIProjectClient(...) as client:
    try:
        agent = await client.agents.create_agent(...)
    except HttpResponseError as e:
        print(f"HTTP Error: {e.status_code}")
        print(f"Message: {e.message}")
    except Exception as e:
        print(f"Unexpected error: {e}")
```

## Context Manager Best Practices

```python
# RECOMMENDED: Use nested context managers
async with (
    DefaultAzureCredential() as credential,
    AIProjectClient(endpoint=endpoint, credential=credential) as client,
):
    # Both credential and client are properly managed
    pass

# ALSO OK: Sequential context managers
async with DefaultAzureCredential() as credential:
    async with AIProjectClient(endpoint=endpoint, credential=credential) as client:
        pass

# AVOID: Manual resource management
client = AIProjectClient(endpoint=endpoint, credential=credential)
try:
    # ... operations
finally:
    await client.close()  # Easy to forget
```

## Async Memory Store Operations

```python
from azure.ai.projects.models import ItemParam, MemoryStoreUpdateCompletedResult
from azure.core.polling import AsyncLROPoller

async with AIProjectClient(...) as client:
    poller: AsyncLROPoller[MemoryStoreUpdateCompletedResult] = (
        await client.memory_stores.begin_update_memories(
            name="conversation-memory",
            scope="user123",
            items=[
                ItemParam(role="user", content="Hello!"),
                ItemParam(role="assistant", content="Hi there!"),
            ],
            previous_update_id=None,
            update_delay=300,
        )
    )
    result = await poller.result()
    print(f"Memory updated: {result}")
```

# Built-in Evaluators Reference

Complete reference for Microsoft Foundry's built-in evaluators using the `azure-ai-projects` SDK.

## Discovering Evaluators

### List All Built-in Evaluators

```python
from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient

endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]

with (
    DefaultAzureCredential() as credential,
    AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
):
    evaluators = project_client.evaluators.list_latest_versions(type="builtin")
    for e in evaluators:
        print(f"{e.name}: {e.description}")
        print(f"  Categories: {e.categories}")
```

### Get Evaluator Schema

Before using an evaluator, query its schema to discover required inputs:

```python
evaluator = project_client.evaluators.get_version(
    name="builtin.task_adherence",
    version="latest"
)
print(f"Init Parameters: {evaluator.definition.init_parameters}")
print(f"Data Schema: {evaluator.definition.data_schema}")
print(f"Metrics: {evaluator.definition.metrics}")
```

## Using Built-in Evaluators

All built-in evaluators use the `azure_ai_evaluator` type with `builtin.` prefix:

```python
testing_criteria = [
    {
        "type": "azure_ai_evaluator",
        "name": "my_coherence_check",          # Your custom name for results
        "evaluator_name": "builtin.coherence", # The actual evaluator
        "data_mapping": {
            "query": "{{item.query}}",
            "response": "{{item.response}}"
        },
        "initialization_parameters": {
            "deployment_name": "gpt-4o-mini"   # Required for LLM-based evaluators
        }
    }
]
```

## Quality Evaluators

### builtin.coherence

Measures logical flow and consistency of the response.

```python
{
    "type": "azure_ai_evaluator",
    "name": "coherence",
    "evaluator_name": "builtin.coherence",
    "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

**Inputs:** query, response  
**Output:** Score 1-5 (5 = highly coherent)

### builtin.fluency

Measures grammatical correctness and natural language quality.

```python
{
    "type": "azure_ai_evaluator",
    "name": "fluency",
    "evaluator_name": "builtin.fluency",
    "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

**Inputs:** query, response  
**Output:** Score 1-5 (5 = perfectly fluent)

### builtin.relevance

Measures how well the response addresses the query given context.

```python
{
    "type": "azure_ai_evaluator",
    "name": "relevance",
    "evaluator_name": "builtin.relevance",
    "data_mapping": {
        "query": "{{item.query}}",
        "response": "{{item.response}}",
        "context": "{{item.context}}"
    },
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

**Inputs:** query, response, context  
**Output:** Score 1-5 (5 = highly relevant)

### builtin.groundedness

Measures whether the response is factually grounded in the provided context.

```python
{
    "type": "azure_ai_evaluator",
    "name": "groundedness",
    "evaluator_name": "builtin.groundedness",
    "data_mapping": {
        "query": "{{item.query}}",
        "response": "{{item.response}}",
        "context": "{{item.context}}"
    },
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

**Inputs:** query, response, context  
**Output:** Score 1-5 (5 = fully grounded)

### builtin.response_completeness

Measures whether the response fully addresses all aspects of the query.

```python
{
    "type": "azure_ai_evaluator",
    "name": "response_completeness",
    "evaluator_name": "builtin.response_completeness",
    "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

**Inputs:** query, response  
**Output:** Score 1-5

## Safety Evaluators

Safety evaluators detect harmful content. They don't require `deployment_name`.

### builtin.violence

Detects violent content.

```python
{
    "type": "azure_ai_evaluator",
    "name": "violence",
    "evaluator_name": "builtin.violence",
    "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}
}
```

**Inputs:** query, response  
**Output:** pass/fail with severity score

### builtin.sexual

Detects inappropriate sexual content.

```python
{
    "type": "azure_ai_evaluator",
    "name": "sexual",
    "evaluator_name": "builtin.sexual",
    "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}
}
```

### builtin.self_harm

Detects content promoting or describing self-harm.

```python
{
    "type": "azure_ai_evaluator",
    "name": "self_harm",
    "evaluator_name": "builtin.self_harm",
    "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}
}
```

### builtin.hate_unfairness

Detects biased or hateful content.

```python
{
    "type": "azure_ai_evaluator",
    "name": "hate_unfairness",
    "evaluator_name": "builtin.hate_unfairness",
    "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}
}
```

## Agent Evaluators

Agent evaluators assess AI agent behavior and tool usage.

### builtin.task_adherence

Evaluates whether the agent follows its system instructions.

```python
{
    "type": "azure_ai_evaluator",
    "name": "task_adherence",
    "evaluator_name": "builtin.task_adherence",
    "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"},
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

**Note:** Use `{{sample.output_items}}` for agent responses to include tool call information.

### builtin.intent_resolution

Evaluates whether the agent correctly understood user intent.

```python
{
    "type": "azure_ai_evaluator",
    "name": "intent_resolution",
    "evaluator_name": "builtin.intent_resolution",
    "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"},
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

### builtin.task_completion

Evaluates whether the agent completed the task end-to-end.

```python
{
    "type": "azure_ai_evaluator",
    "name": "task_completion",
    "evaluator_name": "builtin.task_completion",
    "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"},
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

### builtin.tool_call_accuracy

Evaluates whether tool calls are correct (selection + parameters).

```python
{
    "type": "azure_ai_evaluator",
    "name": "tool_call_accuracy",
    "evaluator_name": "builtin.tool_call_accuracy",
    "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"},
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

### builtin.tool_call_success

Evaluates whether tool calls executed without failures.

```python
{
    "type": "azure_ai_evaluator",
    "name": "tool_call_success",
    "evaluator_name": "builtin.tool_call_success",
    "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"}
}
```

### builtin.tool_selection

Evaluates whether the correct tools were selected.

```python
{
    "type": "azure_ai_evaluator",
    "name": "tool_selection",
    "evaluator_name": "builtin.tool_selection",
    "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"},
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

## NLP Evaluators

NLP evaluators compare responses to ground truth without requiring an LLM.

### builtin.f1_score

Token-level F1 score between response and ground truth.

```python
{
    "type": "azure_ai_evaluator",
    "name": "f1",
    "evaluator_name": "builtin.f1_score",
    "data_mapping": {"response": "{{item.response}}", "ground_truth": "{{item.ground_truth}}"}
}
```

**Output:** Score 0-1

### builtin.bleu_score

BLEU score for generation quality.

```python
{
    "type": "azure_ai_evaluator",
    "name": "bleu",
    "evaluator_name": "builtin.bleu_score",
    "data_mapping": {"response": "{{item.response}}", "ground_truth": "{{item.ground_truth}}"}
}
```

### builtin.rouge_score

ROUGE score for summarization quality.

```python
{
    "type": "azure_ai_evaluator",
    "name": "rouge",
    "evaluator_name": "builtin.rouge_score",
    "data_mapping": {"response": "{{item.response}}", "ground_truth": "{{item.ground_truth}}"}
}
```

### builtin.similarity

Semantic similarity between response and ground truth.

```python
{
    "type": "azure_ai_evaluator",
    "name": "similarity",
    "evaluator_name": "builtin.similarity",
    "data_mapping": {
        "query": "{{item.query}}",
        "response": "{{item.response}}",
        "ground_truth": "{{item.ground_truth}}"
    },
    "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
}
```

## Evaluator Sets by Use Case

### Quick Health Check

```python
testing_criteria = [
    {"type": "azure_ai_evaluator", "name": "coherence", "evaluator_name": "builtin.coherence", ...},
    {"type": "azure_ai_evaluator", "name": "fluency", "evaluator_name": "builtin.fluency", ...},
    {"type": "azure_ai_evaluator", "name": "violence", "evaluator_name": "builtin.violence", ...},
]
```

### Safety Audit

```python
testing_criteria = [
    {"type": "azure_ai_evaluator", "name": "violence", "evaluator_name": "builtin.violence", ...},
    {"type": "azure_ai_evaluator", "name": "sexual", "evaluator_name": "builtin.sexual", ...},
    {"type": "azure_ai_evaluator", "name": "self_harm", "evaluator_name": "builtin.self_harm", ...},
    {"type": "azure_ai_evaluator", "name": "hate_unfairness", "evaluator_name": "builtin.hate_unfairness", ...},
]
```

### Agent Evaluation

```python
testing_criteria = [
    {"type": "azure_ai_evaluator", "name": "task_adherence", "evaluator_name": "builtin.task_adherence", ...},
    {"type": "azure_ai_evaluator", "name": "intent_resolution", "evaluator_name": "builtin.intent_resolution", ...},
    {"type": "azure_ai_evaluator", "name": "tool_call_accuracy", "evaluator_name": "builtin.tool_call_accuracy", ...},
]
```

### RAG Evaluation

```python
testing_criteria = [
    {"type": "azure_ai_evaluator", "name": "groundedness", "evaluator_name": "builtin.groundedness", ...},
    {"type": "azure_ai_evaluator", "name": "relevance", "evaluator_name": "builtin.relevance", ...},
    {"type": "azure_ai_evaluator", "name": "response_completeness", "evaluator_name": "builtin.response_completeness", ...},
]
```

## Data Mapping Reference

| Data Source | Response Mapping | Use Case |
|-------------|------------------|----------|
| JSONL dataset | `{{item.response}}` | Pre-recorded query/response pairs |
| Agent target | `{{sample.output_text}}` | Plain text response |
| Agent target | `{{sample.output_items}}` | Structured JSON with tool calls |

**When to use `sample.output_items`:**
- Tool-related evaluators (tool_call_accuracy, tool_selection, etc.)
- Task adherence evaluator
- Any evaluator needing tool call context

## Related Documentation

- [Azure AI Projects Samples](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/ai/azure-ai-projects/samples/evaluations)
- [Agent Evaluators](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/evaluation-evaluators/agent-evaluators)
- [RAG Evaluators](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/evaluation-evaluators/rag-evaluators)
- [Risk and Safety Evaluators](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/evaluation-evaluators/risk-safety-evaluators)

# Connections Operations Reference

## Overview

Connections provide access to external Azure services like Azure OpenAI, Azure AI Search, Bing, and more.

## List Connections

### List All Connections

```python
connections = project_client.connections.list()
for conn in connections:
    print(f"Name: {conn.name}")
    print(f"Type: {conn.connection_type}")
    print(f"ID: {conn.id}")
    print("---")
```

### Filter by Connection Type

```python
from azure.ai.projects.models import ConnectionType

# List Azure OpenAI connections
for conn in project_client.connections.list(
    connection_type=ConnectionType.AZURE_OPEN_AI
):
    print(f"Azure OpenAI: {conn.name}")

# List Azure AI Search connections
for conn in project_client.connections.list(
    connection_type=ConnectionType.AZURE_AI_SEARCH
):
    print(f"AI Search: {conn.name}")
```

## Connection Types

```python
from azure.ai.projects.models import ConnectionType

# Available connection types:
# - ConnectionType.AZURE_OPEN_AI
# - ConnectionType.AZURE_AI_SEARCH
# - ConnectionType.AZURE_BLOB
# - ConnectionType.AZURE_AI_SERVICES
# - ConnectionType.API_KEY
# - ConnectionType.COGNITIVE_SEARCH
# - ConnectionType.COGNITIVE_SERVICE
# - ConnectionType.CUSTOM
```

## Get Connection

### Get by Name

```python
connection = project_client.connections.get(connection_name="my-search-connection")
print(f"Name: {connection.name}")
print(f"Type: {connection.connection_type}")
```

### Get with Credentials

```python
connection = project_client.connections.get(
    connection_name="my-search-connection",
    include_credentials=True,
)
print(f"Endpoint: {connection.endpoint_url}")
# Access credentials based on connection type
```

### Get Default Connection

```python
from azure.ai.projects.models import ConnectionType

# Get default Azure OpenAI connection
default_aoai = project_client.connections.get_default(
    connection_type=ConnectionType.AZURE_OPEN_AI
)
print(f"Default Azure OpenAI: {default_aoai.name}")

# Get with credentials
default_aoai = project_client.connections.get_default(
    connection_type=ConnectionType.AZURE_OPEN_AI,
    include_credentials=True,
)
```

## Using Connections with Tools

### Bing Grounding

```python
from azure.ai.projects.models import (
    BingGroundingAgentTool,
    BingGroundingSearchToolParameters,
    BingGroundingSearchConfiguration,
)

# Get Bing connection
bing_connection = project_client.connections.get(
    os.environ["BING_CONNECTION_NAME"]
)

# Use in agent
tools = [
    BingGroundingAgentTool(
        bing_grounding=BingGroundingSearchToolParameters(
            search_configurations=[
                BingGroundingSearchConfiguration(
                    project_connection_id=bing_connection.id
                )
            ]
        )
    )
]
```

### Azure AI Search

```python
from azure.ai.projects.models import (
    AzureAISearchAgentTool,
    AzureAISearchToolResource,
    AISearchIndexResource,
    AzureAISearchQueryType,
)

# Get search connection
search_connection = project_client.connections.get(
    os.environ["AI_SEARCH_CONNECTION_NAME"]
)

# Use in agent
tools = [
    AzureAISearchAgentTool(
        azure_ai_search=AzureAISearchToolResource(
            indexes=[
                AISearchIndexResource(
                    project_connection_id=search_connection.id,
                    index_name="my-index",
                    query_type=AzureAISearchQueryType.SEMANTIC,
                )
            ]
        )
    )
]
```

### OpenAI Client from Connection

```python
# Get OpenAI client for a specific Azure OpenAI connection
openai_client = project_client.get_openai_client(
    api_version="2024-10-21",
    connection_name="my-aoai-connection",
)

response = openai_client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Hello!"}],
)
```

## Connection Properties

```python
connection = project_client.connections.get(
    connection_name="my-connection",
    include_credentials=True,
)

# Common properties
print(f"ID: {connection.id}")
print(f"Name: {connection.name}")
print(f"Type: {connection.connection_type}")
print(f"Endpoint: {connection.endpoint_url}")

# Check authentication type
if hasattr(connection, 'authentication_type'):
    print(f"Auth Type: {connection.authentication_type}")
```

## Environment Variables Pattern

```bash
# Recommended environment variables for connections
BING_CONNECTION_NAME=my-bing-connection
AI_SEARCH_CONNECTION_NAME=my-search-connection
AOAI_CONNECTION_NAME=my-aoai-connection
```

```python
import os

# Load connections from environment
bing_conn = project_client.connections.get(os.environ["BING_CONNECTION_NAME"])
search_conn = project_client.connections.get(os.environ["AI_SEARCH_CONNECTION_NAME"])
```

# Custom Evaluators Reference

Create custom evaluators when built-in evaluators don't meet your needs using the `azure-ai-projects` SDK.

## Evaluator Types

| Type | Best For | Requires LLM |
|------|----------|--------------|
| **Code-based** | Pattern matching, format validation, deterministic rules | No |
| **Prompt-based** | Subjective judgment, semantic analysis, nuanced evaluation | Yes |

## Code-Based Evaluators

Use Python code for deterministic evaluation logic.

### Basic Code Evaluator

```python
from azure.ai.projects import AIProjectClient
from azure.ai.projects.models import (
    EvaluatorVersion,
    EvaluatorCategory,
    EvaluatorType,
    CodeBasedEvaluatorDefinition,
    EvaluatorMetric,
    EvaluatorMetricType,
    EvaluatorMetricDirection,
)
from azure.identity import DefaultAzureCredential
import os

endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]

with (
    DefaultAzureCredential() as credential,
    AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
):
    evaluator = project_client.evaluators.create_version(
        name="word_count_evaluator",
        evaluator_version=EvaluatorVersion(
            evaluator_type=EvaluatorType.CUSTOM,
            categories=[EvaluatorCategory.QUALITY],
            display_name="Word Count",
            description="Counts words in response and checks for conciseness",
            definition=CodeBasedEvaluatorDefinition(
                code_text='''
def grade(sample, item) -> dict:
    response = item.get("response", "")
    word_count = len(response.split())
    return {
        "word_count": word_count,
        "is_concise": word_count < 100
    }
''',
                data_schema={
                    "type": "object",
                    "properties": {
                        "response": {"type": "string"}
                    },
                    "required": ["response"]
                },
                metrics={
                    "word_count": EvaluatorMetric(
                        type=EvaluatorMetricType.ORDINAL,
                        desirable_direction=EvaluatorMetricDirection.DECREASE,
                        min_value=0,
                        max_value=10000,
                    ),
                    "is_concise": EvaluatorMetric(
                        type=EvaluatorMetricType.BINARY,
                    ),
                },
            ),
        ),
    )
    print(f"Created evaluator: {evaluator.name} (version {evaluator.version})")
```

### Code Evaluator: Keyword Checker

```python
evaluator = project_client.evaluators.create_version(
    name="disclaimer_checker",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Disclaimer Checker",
        description="Verifies required disclaimers are present in response",
        definition=CodeBasedEvaluatorDefinition(
            code_text='''
def grade(sample, item) -> dict:
    response = item.get("response", "").lower()
    required_keywords = ["disclaimer", "not financial advice", "consult a professional"]
    
    found = [kw for kw in required_keywords if kw in response]
    missing = [kw for kw in required_keywords if kw not in response]
    
    score = len(found) / len(required_keywords) if required_keywords else 1.0
    
    return {
        "compliance_score": score,
        "missing_disclaimers": ", ".join(missing) if missing else "none",
        "passes": score >= 0.8
    }
''',
            data_schema={
                "type": "object",
                "properties": {"response": {"type": "string"}},
                "required": ["response"]
            },
            metrics={
                "compliance_score": EvaluatorMetric(
                    type=EvaluatorMetricType.ORDINAL,
                    desirable_direction=EvaluatorMetricDirection.INCREASE,
                    min_value=0.0,
                    max_value=1.0,
                ),
                "passes": EvaluatorMetric(type=EvaluatorMetricType.BINARY),
            },
        ),
    ),
)
```

### Code Evaluator: JSON Format Validator

```python
evaluator = project_client.evaluators.create_version(
    name="json_format_checker",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="JSON Format Validator",
        description="Checks if response is valid JSON with required fields",
        definition=CodeBasedEvaluatorDefinition(
            code_text='''
import json

def grade(sample, item) -> dict:
    response = item.get("response", "")
    required_fields = item.get("required_fields", [])
    
    try:
        parsed = json.loads(response)
        is_valid_json = True
        
        if required_fields:
            missing = [f for f in required_fields if f not in parsed]
            has_required_fields = len(missing) == 0
        else:
            has_required_fields = True
            missing = []
            
    except json.JSONDecodeError:
        is_valid_json = False
        has_required_fields = False
        missing = required_fields
    
    return {
        "is_valid_json": is_valid_json,
        "has_required_fields": has_required_fields,
        "missing_fields": ", ".join(missing) if missing else "none"
    }
''',
            data_schema={
                "type": "object",
                "properties": {
                    "response": {"type": "string"},
                    "required_fields": {"type": "array", "items": {"type": "string"}}
                },
                "required": ["response"]
            },
            metrics={
                "is_valid_json": EvaluatorMetric(type=EvaluatorMetricType.BINARY),
                "has_required_fields": EvaluatorMetric(type=EvaluatorMetricType.BINARY),
            },
        ),
    ),
)
```

## Prompt-Based Evaluators

Use LLM judgment for subjective evaluation.

### Basic Prompt Evaluator

```python
from azure.ai.projects.models import PromptBasedEvaluatorDefinition

evaluator = project_client.evaluators.create_version(
    name="helpfulness_evaluator",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Helpfulness Evaluator",
        description="Evaluates how helpful the response is to the user",
        definition=PromptBasedEvaluatorDefinition(
            prompt_text='''
You are an expert evaluator. Rate the helpfulness of the AI assistant's response.

Query: {query}
Response: {response}

Scoring (1-5):
1 = Not helpful at all, doesn't address the query
2 = Slightly helpful, partially addresses the query
3 = Moderately helpful, addresses most of the query
4 = Very helpful, fully addresses the query
5 = Extremely helpful, exceeds expectations

Return ONLY valid JSON: {"score": <1-5>, "reason": "<brief explanation>"}
''',
            init_parameters={
                "type": "object",
                "properties": {
                    "deployment_name": {"type": "string", "description": "Model deployment name"}
                },
                "required": ["deployment_name"]
            },
            data_schema={
                "type": "object",
                "properties": {
                    "query": {"type": "string"},
                    "response": {"type": "string"}
                },
                "required": ["query", "response"]
            },
            metrics={
                "score": EvaluatorMetric(
                    type=EvaluatorMetricType.ORDINAL,
                    desirable_direction=EvaluatorMetricDirection.INCREASE,
                    min_value=1,
                    max_value=5,
                ),
            },
        ),
    ),
)
```

### Prompt Evaluator: Brand Tone Checker

```python
evaluator = project_client.evaluators.create_version(
    name="brand_tone_checker",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Brand Tone Checker",
        description="Evaluates if response matches company brand voice guidelines",
        definition=PromptBasedEvaluatorDefinition(
            prompt_text='''
You are evaluating if an AI assistant's response matches brand voice guidelines.

Brand Guidelines:
- Professional but friendly
- Avoid jargon, use simple language
- Always offer next steps or additional help
- Never use negative language about competitors
- End with a helpful call-to-action

Response to evaluate:
{response}

Score the response from 1-5:
5 = Perfectly matches brand voice
4 = Mostly matches, minor issues
3 = Partially matches
2 = Significant tone issues
1 = Does not match brand voice

Return ONLY valid JSON: {"score": <1-5>, "reason": "<brief explanation>", "suggestions": "<improvement suggestions>"}
''',
            init_parameters={
                "type": "object",
                "properties": {"deployment_name": {"type": "string"}},
                "required": ["deployment_name"]
            },
            data_schema={
                "type": "object",
                "properties": {"response": {"type": "string"}},
                "required": ["response"]
            },
            metrics={
                "score": EvaluatorMetric(
                    type=EvaluatorMetricType.ORDINAL,
                    min_value=1,
                    max_value=5,
                ),
            },
        ),
    ),
)
```

### Prompt Evaluator: Factual Accuracy

```python
evaluator = project_client.evaluators.create_version(
    name="factual_accuracy_checker",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Factual Accuracy",
        description="Checks if response claims are supported by context",
        definition=PromptBasedEvaluatorDefinition(
            prompt_text='''
Evaluate whether the response contains only facts supported by the provided context.

Context (source of truth):
{context}

Response to evaluate:
{response}

Analysis steps:
1. Identify each factual claim in the response
2. Check if each claim is supported by the context
3. Note any unsupported or fabricated claims

Scoring (1-5):
1 = Mostly fabricated or incorrect
2 = Many unsupported claims
3 = Mixed: some facts but notable errors
4 = Mostly factual, minor issues
5 = Fully factual, no unsupported claims

Return ONLY valid JSON: {"score": <1-5>, "reason": "<explanation>", "unsupported_claims": ["<list of unsupported claims>"]}
''',
            init_parameters={
                "type": "object",
                "properties": {"deployment_name": {"type": "string"}},
                "required": ["deployment_name"]
            },
            data_schema={
                "type": "object",
                "properties": {
                    "context": {"type": "string"},
                    "response": {"type": "string"}
                },
                "required": ["context", "response"]
            },
            metrics={
                "score": EvaluatorMetric(
                    type=EvaluatorMetricType.ORDINAL,
                    min_value=1,
                    max_value=5,
                ),
            },
        ),
    ),
)
```

## Using Custom Evaluators

### In Testing Criteria

```python
testing_criteria = [
    # Built-in evaluator
    {
        "type": "azure_ai_evaluator",
        "name": "coherence",
        "evaluator_name": "builtin.coherence",
        "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
        "initialization_parameters": {"deployment_name": "gpt-4o-mini"}
    },
    # Custom code-based evaluator
    {
        "type": "azure_ai_evaluator",
        "name": "word_count",
        "evaluator_name": "word_count_evaluator",
        "data_mapping": {"response": "{{item.response}}"}
    },
    # Custom prompt-based evaluator
    {
        "type": "azure_ai_evaluator",
        "name": "helpfulness",
        "evaluator_name": "helpfulness_evaluator",
        "initialization_parameters": {"deployment_name": "gpt-4o-mini"},
        "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"}
    },
]

eval_object = openai_client.evals.create(
    name="Mixed Evaluators Test",
    data_source_config=data_source_config,
    testing_criteria=testing_criteria,
)
```

## Managing Custom Evaluators

### List Custom Evaluators

```python
evaluators = project_client.evaluators.list_latest_versions(type="custom")
for e in evaluators:
    print(f"{e.name} (v{e.version}): {e.display_name}")
```

### Get Evaluator Details

```python
evaluator = project_client.evaluators.get_version(
    name="helpfulness_evaluator",
    version="latest"
)
print(f"Data Schema: {evaluator.definition.data_schema}")
print(f"Metrics: {evaluator.definition.metrics}")
```

### Update Evaluator

```python
updated = project_client.evaluators.update_version(
    name="word_count_evaluator",
    version="1",
    evaluator_version={
        "description": "Updated description",
        "display_name": "Word Count v2",
    }
)
```

### Delete Evaluator

```python
project_client.evaluators.delete_version(
    name="word_count_evaluator",
    version="1"
)
```

## Best Practices

1. **Use code-based for deterministic logic** - Pattern matching, format validation, keyword checking
2. **Use prompt-based for subjective judgment** - Quality assessment, tone evaluation, semantic analysis
3. **Always define data_schema** - Ensures correct data mapping
4. **Define meaningful metrics** - Use appropriate types (ORDINAL, BINARY)
5. **Test before production** - Run evaluator on sample data first
6. **Version your evaluators** - Create new versions instead of modifying existing ones

## Related Documentation

- [Custom Evaluators](https://learn.microsoft.com/en-us/azure/ai-foundry/concepts/evaluation-evaluators/custom-evaluators)
- [Code-based evaluator sample](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_catalog_code_based_evaluators.py)
- [Prompt-based evaluator sample](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/ai/azure-ai-projects/samples/evaluations/sample_eval_catalog_prompt_based_evaluators.py)

# Datasets and Indexes Reference

## Datasets

### Upload File

```python
from azure.ai.projects.models import DatasetVersion

dataset = project_client.datasets.upload_file(
    name="my-dataset",
    version="1.0",
    file_path="./data/training_data.csv",
    connection_name="my-storage-connection",
)
print(f"Dataset uploaded: {dataset.name} v{dataset.version}")
```

### Upload Folder

```python
import re
from azure.ai.projects.models import DatasetVersion

dataset = project_client.datasets.upload_folder(
    name="document-collection",
    version="2.0",
    folder="./data/documents/",
    connection_name="my-storage-connection",
    file_pattern=re.compile(r"\.(txt|csv|md|json)$", re.IGNORECASE),
)
print(f"Folder uploaded: {dataset.name} v{dataset.version}")
```

### Get Dataset

```python
dataset = project_client.datasets.get(name="my-dataset", version="1.0")
print(f"Name: {dataset.name}")
print(f"Version: {dataset.version}")
```

### Get Dataset Credentials

```python
credentials = project_client.datasets.get_credentials(
    name="my-dataset",
    version="1.0",
)
# Use credentials to access dataset storage
```

### List Datasets

```python
# List all datasets
for dataset in project_client.datasets.list():
    print(f"{dataset.name}: {dataset.version}")

# List versions of a specific dataset
for dataset in project_client.datasets.list_versions(name="my-dataset"):
    print(f"Version: {dataset.version}")
```

### Delete Dataset

```python
project_client.datasets.delete(name="my-dataset", version="1.0")
```

## Indexes

### Create or Update Index

```python
from azure.ai.projects.models import AzureAISearchIndex

index = project_client.indexes.create_or_update(
    name="my-index",
    version="1.0",
    index=AzureAISearchIndex(
        connection_name="my-ai-search-connection",
        index_name="products-index",
    ),
)
print(f"Index created: {index.name} v{index.version}")
```

### Get Index

```python
index = project_client.indexes.get(name="my-index", version="1.0")
print(f"Name: {index.name}")
print(f"Version: {index.version}")
```

### List Indexes

```python
# List all indexes
for index in project_client.indexes.list():
    print(f"{index.name}: {index.version}")

# List versions of a specific index
for index in project_client.indexes.list_versions(name="my-index"):
    print(f"Version: {index.version}")
```

### Delete Index

```python
project_client.indexes.delete(name="my-index", version="1.0")
```

## Using Indexes with Agents

```python
from azure.ai.projects.models import (
    AzureAISearchAgentTool,
    AzureAISearchToolResource,
    AISearchIndexResource,
    AzureAISearchQueryType,
    PromptAgentDefinition,
)

# Create index reference
index = project_client.indexes.get(name="products-index", version="1.0")

# Get connection for the index
search_connection = project_client.connections.get("my-ai-search-connection")

# Create agent with index
agent = project_client.agents.create_version(
    agent_name="search-agent",
    definition=PromptAgentDefinition(
        model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
        instructions="Search the product catalog to answer questions.",
        tools=[
            AzureAISearchAgentTool(
                azure_ai_search=AzureAISearchToolResource(
                    indexes=[
                        AISearchIndexResource(
                            project_connection_id=search_connection.id,
                            index_name="products-index",
                            query_type=AzureAISearchQueryType.SEMANTIC,
                        )
                    ]
                )
            )
        ],
    ),
)
```

## Version Management Pattern

```python
# Semantic versioning for datasets
dataset_v1 = project_client.datasets.upload_file(
    name="training-data",
    version="1.0.0",
    file_path="./v1/data.csv",
    connection_name="storage",
)

# Update with new version
dataset_v2 = project_client.datasets.upload_file(
    name="training-data",
    version="1.1.0",  # Minor version bump
    file_path="./v2/data.csv",
    connection_name="storage",
)

# List all versions
versions = list(project_client.datasets.list_versions(name="training-data"))
print(f"Available versions: {[v.version for v in versions]}")
```

# Deployments Operations Reference

## Overview

Deployments represent AI model deployments in your Azure AI Foundry project.

## List Deployments

### List All Deployments

```python
deployments = project_client.deployments.list()
for deployment in deployments:
    print(f"Name: {deployment.name}")
    print(f"Model: {deployment.model_name}")
    print(f"Publisher: {deployment.model_publisher}")
    print("---")
```

### Filter by Publisher

```python
# List only OpenAI model deployments
for deployment in project_client.deployments.list(model_publisher="OpenAI"):
    print(f"{deployment.name}: {deployment.model_name}")
```

### Filter by Model Name

```python
# List deployments of a specific model
for deployment in project_client.deployments.list(model_name="gpt-4o"):
    print(f"{deployment.name}: {deployment.model_version}")
```

## Get Deployment

```python
from azure.ai.projects.models import ModelDeployment

deployment = project_client.deployments.get("my-deployment-name")

if isinstance(deployment, ModelDeployment):
    print(f"Type: {deployment.type}")
    print(f"Name: {deployment.name}")
    print(f"Model Name: {deployment.model_name}")
    print(f"Model Version: {deployment.model_version}")
    print(f"Model Publisher: {deployment.model_publisher}")
    print(f"Capabilities: {deployment.capabilities}")
```

## Deployment Properties

```python
deployment = project_client.deployments.get("gpt-4o-mini")

# Available properties
print(f"Name: {deployment.name}")           # Deployment name
print(f"Model: {deployment.model_name}")    # e.g., "gpt-4o-mini"
print(f"Version: {deployment.model_version}")  # e.g., "2024-07-18"
print(f"Publisher: {deployment.model_publisher}")  # e.g., "OpenAI"
print(f"Type: {deployment.type}")           # Deployment type
print(f"Capabilities: {deployment.capabilities}")  # Model capabilities
```

## Using Deployments

### Dynamic Model Selection

```python
# Find available GPT-4 deployments
gpt4_deployments = [
    d for d in project_client.deployments.list()
    if "gpt-4" in d.model_name.lower()
]

if gpt4_deployments:
    deployment_name = gpt4_deployments[0].name
    
    agent = project_client.agents.create_agent(
        model=deployment_name,
        name="dynamic-agent",
        instructions="You are helpful.",
    )
```

### Capability Checking

```python
deployment = project_client.deployments.get("my-deployment")

# Check if deployment supports certain capabilities
if deployment.capabilities:
    supports_vision = deployment.capabilities.get("vision", False)
    supports_functions = deployment.capabilities.get("function_calling", False)
    
    print(f"Vision: {supports_vision}")
    print(f"Function Calling: {supports_functions}")
```

## Environment Variables Pattern

```bash
# Store deployment name in environment
AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o-mini
```

```python
import os

# Use deployment from environment
agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="my-agent",
    instructions="You are helpful.",
)
```

## List Available Models

```python
# Print all available models grouped by publisher
from collections import defaultdict

deployments_by_publisher = defaultdict(list)

for deployment in project_client.deployments.list():
    deployments_by_publisher[deployment.model_publisher].append(deployment)

for publisher, deployments in deployments_by_publisher.items():
    print(f"\n{publisher}:")
    for d in deployments:
        print(f"  - {d.name} ({d.model_name} v{d.model_version})")
```

# Evaluation Operations Reference

Evaluate AI agents and models using Microsoft Foundry's cloud evaluation service.

## Setup

```python
import os
from azure.ai.projects import AIProjectClient
from azure.identity import DefaultAzureCredential

endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]
deployment = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o-mini")

with (
    DefaultAzureCredential() as credential,
    AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
):
    openai_client = project_client.get_openai_client()
    # Use openai_client.evals.*
```

## Quick Start: Run a Basic Evaluation

```python
from openai.types.evals.create_eval_jsonl_run_data_source_param import (
    CreateEvalJSONLRunDataSourceParam,
    SourceFileContent,
    SourceFileContentContent,
)
from openai.types.eval_create_params import DataSourceConfigCustom

# 1. Prepare test data
data = [
    {"query": "What is Azure?", "response": "Azure is Microsoft's cloud platform."},
    {"query": "What is AI?", "response": "AI is artificial intelligence."},
]

# 2. Create data source
data_source = CreateEvalJSONLRunDataSourceParam(
    type="jsonl",
    source=SourceFileContent(
        type="file_content",
        content=[SourceFileContentContent(item=item, sample={}) for item in data],
    ),
)

# 3. Configure schema
data_source_config = DataSourceConfigCustom(
    type="custom",
    item_schema={
        "type": "object",
        "properties": {
            "query": {"type": "string"},
            "response": {"type": "string"},
        },
        "required": ["query", "response"],
    },
    include_sample_schema=False,
)

# 4. Define evaluators
testing_criteria = [
    {
        "type": "azure_ai_evaluator",
        "name": "coherence",
        "evaluator_name": "builtin.coherence",
        "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
        "initialization_parameters": {"deployment_name": deployment},
    },
    {
        "type": "azure_ai_evaluator",
        "name": "relevance",
        "evaluator_name": "builtin.relevance",
        "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
        "initialization_parameters": {"deployment_name": deployment},
    },
]

# 5. Create and run evaluation
eval_object = openai_client.evals.create(
    name="Quality Evaluation",
    data_source_config=data_source_config,
    testing_criteria=testing_criteria,
)

run = openai_client.evals.runs.create(
    eval_id=eval_object.id,
    name="Run 1",
    data_source=data_source,
)

# 6. Poll for completion
import time
while run.status not in ["completed", "failed", "cancelled"]:
    time.sleep(5)
    run = openai_client.evals.runs.retrieve(eval_id=eval_object.id, run_id=run.id)
    print(f"Status: {run.status}")

# 7. Retrieve results
output_items = list(openai_client.evals.runs.output_items.list(
    eval_id=eval_object.id, run_id=run.id
))

for item in output_items:
    for result in item.results:
        print(f"{result.name}: {result.score}")
```

## Built-in Evaluators

Use the `builtin.` prefix for all built-in evaluators:

### Quality Evaluators

| Evaluator | Data Mapping | Use Case |
|-----------|--------------|----------|
| `builtin.coherence` | query, response | Logical flow and consistency |
| `builtin.relevance` | query, response | Response addresses the query |
| `builtin.fluency` | query, response | Language quality and readability |
| `builtin.groundedness` | query, context, response | Factual alignment with context |

### Safety Evaluators

| Evaluator | Data Mapping | Use Case |
|-----------|--------------|----------|
| `builtin.violence` | query, response | Violent content detection |
| `builtin.sexual` | query, response | Sexual content detection |
| `builtin.self_harm` | query, response | Self-harm content detection |
| `builtin.hate_unfairness` | query, response | Hate/bias detection |

### Agent Evaluators

| Evaluator | Data Mapping | Use Case |
|-----------|--------------|----------|
| `builtin.intent_resolution` | query, response | Did agent understand intent? |
| `builtin.response_completeness` | query, response | Did agent answer fully? |
| `builtin.task_adherence` | query, response | Did agent follow instructions? |
| `builtin.tool_call_accuracy` | query, response (JSON) | Were tool calls correct? |

See [built-in-evaluators.md](built-in-evaluators.md) for complete evaluator reference.

## Agent Evaluation

For evaluating AI agents with tool calls, use `sample` mapping:

```python
# Data with agent outputs
data_source = CreateEvalJSONLRunDataSourceParam(
    type="jsonl",
    source=SourceFileContent(
        type="file_content",
        content=[
            SourceFileContentContent(
                item={"query": "Weather in Seattle?"},
                sample={
                    "output_text": "It's 55°F and cloudy in Seattle.",
                    "output_items": [
                        {
                            "type": "tool_call",
                            "name": "get_weather",
                            "arguments": {"location": "Seattle"},
                            "result": {"temp": "55", "condition": "cloudy"},
                        }
                    ],
                },
            )
        ],
    ),
)

data_source_config = DataSourceConfigCustom(
    type="custom",
    item_schema={"type": "object", "properties": {"query": {"type": "string"}}},
    include_sample_schema=True,  # Required for agent evaluations
)

testing_criteria = [
    {
        "type": "azure_ai_evaluator",
        "name": "intent_resolution",
        "evaluator_name": "builtin.intent_resolution",
        "data_mapping": {
            "query": "{{item.query}}",
            "response": "{{sample.output_text}}",  # Use sample for agent outputs
        },
        "initialization_parameters": {"deployment_name": deployment},
    },
    {
        "type": "azure_ai_evaluator",
        "name": "tool_call_accuracy",
        "evaluator_name": "builtin.tool_call_accuracy",
        "data_mapping": {
            "query": "{{item.query}}",
            "response": "{{sample.output_items}}",  # JSON with tool calls
        },
        "initialization_parameters": {"deployment_name": deployment},
    },
]
```

## OpenAI Graders

For simpler evaluation patterns, use OpenAI graders:

```python
testing_criteria = [
    # Label grader (classification)
    {
        "type": "label_model",
        "name": "sentiment",
        "model": deployment,
        "input": [{"role": "user", "content": "Classify sentiment: {{item.response}}"}],
        "labels": ["positive", "negative", "neutral"],
        "passing_labels": ["positive", "neutral"],
    },
    # String check grader
    {
        "type": "string_check",
        "name": "has_disclaimer",
        "input": "{{item.response}}",
        "operation": "contains",
        "reference": "Please consult",
    },
    # Text similarity grader
    {
        "type": "text_similarity",
        "name": "matches_expected",
        "input": "{{item.response}}",
        "reference": "{{item.expected}}",
        "evaluation_metric": "fuzzy_match",
        "pass_threshold": 0.8,
    },
]
```

## Custom Evaluators

Create custom evaluators for domain-specific needs.

### Code-Based Evaluator

```python
from azure.ai.projects.models import (
    EvaluatorVersion, EvaluatorCategory, EvaluatorType,
    CodeBasedEvaluatorDefinition, EvaluatorMetric, EvaluatorMetricType,
)

evaluator = project_client.evaluators.create_version(
    name="word_count",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Word Count",
        definition=CodeBasedEvaluatorDefinition(
            code_text='''
def grade(sample, item) -> dict:
    return {"word_count": len(item.get("response", "").split())}
''',
            data_schema={
                "type": "object",
                "properties": {"response": {"type": "string"}},
                "required": ["response"],
            },
            metrics={
                "word_count": EvaluatorMetric(type=EvaluatorMetricType.ORDINAL),
            },
        ),
    ),
)
```

### Prompt-Based Evaluator

```python
from azure.ai.projects.models import PromptBasedEvaluatorDefinition

evaluator = project_client.evaluators.create_version(
    name="helpfulness",
    evaluator_version=EvaluatorVersion(
        evaluator_type=EvaluatorType.CUSTOM,
        categories=[EvaluatorCategory.QUALITY],
        display_name="Helpfulness",
        definition=PromptBasedEvaluatorDefinition(
            prompt_text='''
Rate the helpfulness of the response (1-5):
Query: {query}
Response: {response}
Return JSON: {"score": <1-5>, "reason": "<explanation>"}
''',
            init_parameters={
                "type": "object",
                "properties": {"deployment_name": {"type": "string"}},
                "required": ["deployment_name"],
            },
            data_schema={
                "type": "object",
                "properties": {"query": {"type": "string"}, "response": {"type": "string"}},
                "required": ["query", "response"],
            },
            metrics={"score": EvaluatorMetric(type=EvaluatorMetricType.ORDINAL)},
        ),
    ),
)
```

See [custom-evaluators.md](custom-evaluators.md) for complete custom evaluator reference.

## Discover Available Evaluators

```python
# List built-in evaluators
evaluators = project_client.evaluators.list_latest_versions(type="builtin")
for e in evaluators:
    print(f"builtin.{e.name}: {e.description}")

# List custom evaluators
custom = project_client.evaluators.list_latest_versions(type="custom")
for e in custom:
    print(f"{e.name}: {e.description}")
```

## Data Mapping Reference

| Pattern | Source | Use Case |
|---------|--------|----------|
| `{{item.field}}` | Your JSONL data | Standard evaluation data |
| `{{sample.output_text}}` | Agent response (text) | Agent text outputs |
| `{{sample.output_items}}` | Agent response (JSON) | Tool calls, structured data |

## CLI Tool

A batch evaluation script is available at `scripts/run_batch_evaluation.py`:

```bash
python run_batch_evaluation.py --data test_data.jsonl --evaluators coherence relevance
python run_batch_evaluation.py --data test_data.jsonl --safety
python run_batch_evaluation.py --data test_data.jsonl --agent --evaluators intent_resolution
```

## Related Reference Files

- [built-in-evaluators.md](built-in-evaluators.md): Complete built-in evaluator reference
- [custom-evaluators.md](custom-evaluators.md): Code and prompt-based evaluator patterns

## Related Documentation

- [Azure AI Projects Evaluation Samples](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/ai/azure-ai-projects/samples/evaluations)
- [Cloud Evaluation Documentation](https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/develop/cloud-evaluation)

# Agent Tools Reference

## Tool Import Patterns

```python
# From azure.ai.agents.models (low-level tools)
from azure.ai.agents.models import (
    CodeInterpreterTool,
    FileSearchTool,
    FunctionTool,
    BingGroundingTool,
    OpenApiTool,
    OpenApiAnonymousAuthDetails,
    FilePurpose,
    MessageAttachment,
    ToolSet,
    SharepointTool,
    FabricTool,
    ConnectedAgentTool,
    McpTool,
)

# From azure.ai.projects.models (project-level tools)
from azure.ai.projects.models import (
    AzureAISearchAgentTool,
    AzureAISearchToolResource,
    AISearchIndexResource,
    AzureAISearchQueryType,
    BingGroundingAgentTool,
    BingGroundingSearchToolParameters,
    BingGroundingSearchConfiguration,
    PromptAgentDefinition,
)
```

## CodeInterpreterTool

Execute Python code in a sandboxed environment.

### Basic Usage

```python
from azure.ai.agents.models import CodeInterpreterTool

code_interpreter = CodeInterpreterTool()

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="code-agent",
    instructions="You can execute Python code. Use Code Interpreter for calculations and visualizations.",
    tools=code_interpreter.definitions,
    tool_resources=code_interpreter.resources,
)
```

### With File Upload

```python
from azure.ai.agents.models import CodeInterpreterTool, FilePurpose

# Upload file for code interpreter
file = project_client.agents.files.upload_and_poll(
    file_path="data.csv",
    purpose=FilePurpose.AGENTS,
)

code_interpreter = CodeInterpreterTool()

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="data-agent",
    instructions="Analyze the uploaded data file.",
    tools=code_interpreter.definitions,
    tool_resources={"code_interpreter": {"file_ids": [file.id]}},
)
```

## FileSearchTool

RAG over uploaded documents using vector stores.

### Basic Usage

```python
from azure.ai.agents.models import FileSearchTool, FilePurpose

# Upload and create vector store
file = project_client.agents.files.upload_and_poll(
    file_path="./data/product_info.md",
    purpose=FilePurpose.AGENTS,
)
vector_store = project_client.agents.vector_stores.create_and_poll(
    file_ids=[file.id],
    name="product-docs",
)

# Create file search tool
file_search = FileSearchTool(vector_store_ids=[vector_store.id])

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="search-agent",
    instructions="Search uploaded files to answer questions.",
    tools=file_search.definitions,
    tool_resources=file_search.resources,
)
```

### With Message Attachment

```python
from azure.ai.agents.models import MessageAttachment, FileSearchTool

attachment = MessageAttachment(
    file_id=file.id,
    tools=FileSearchTool().definitions,
)

message = project_client.agents.messages.create(
    thread_id=thread.id,
    role="user",
    content="What features are mentioned in this document?",
    attachments=[attachment],
)
```

## FunctionTool

Define custom Python functions for agents to call.

### Basic Usage

```python
from azure.ai.agents.models import FunctionTool

def get_weather(location: str) -> str:
    """Get weather for a location."""
    return f"Weather in {location}: Sunny, 72F"

def get_stock_price(symbol: str) -> str:
    """Get current stock price."""
    return f"{symbol}: $150.00"

functions = FunctionTool(functions=[get_weather, get_stock_price])

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="function-agent",
    instructions="Help with weather and stock queries.",
    tools=functions.definitions,
)
```

### With ToolSet and Auto-Execution

```python
from azure.ai.agents.models import FunctionTool, ToolSet

def get_weather(location: str) -> str:
    """Get weather for a location."""
    return f"Weather in {location}: Sunny, 72F"

functions = FunctionTool(functions=[get_weather])
toolset = ToolSet()
toolset.add(functions)

# Enable auto function calls
project_client.agents.enable_auto_function_calls(toolset)

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="auto-function-agent",
    instructions="Help with weather queries.",
    toolset=toolset,
)

# Process run - functions auto-execute
run = project_client.agents.runs.create_and_process(
    thread_id=thread.id,
    agent_id=agent.id,
    toolset=toolset,
)
```

### Explicit Function Definition

```python
from azure.ai.projects.models import FunctionTool

tool = FunctionTool(
    name="get_horoscope",
    parameters={
        "type": "object",
        "properties": {
            "sign": {
                "type": "string",
                "description": "An astrological sign like Taurus or Aquarius",
            },
        },
        "required": ["sign"],
        "additionalProperties": False,
    },
    description="Get today's horoscope for an astrological sign.",
    strict=True,
)
```

## BingGroundingTool

Real-time web search grounding.

### Using Low-Level Tool

```python
from azure.ai.agents.models import BingGroundingTool

conn_id = os.environ["BING_CONNECTION_NAME"]
bing = BingGroundingTool(connection_id=conn_id)

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="bing-agent",
    instructions="Use web search to find current information.",
    tools=bing.definitions,
)
```

### Using Project-Level Tool

```python
from azure.ai.projects.models import (
    PromptAgentDefinition,
    BingGroundingAgentTool,
    BingGroundingSearchToolParameters,
    BingGroundingSearchConfiguration,
)

bing_connection = project_client.connections.get(
    os.environ["BING_PROJECT_CONNECTION_NAME"]
)

agent = project_client.agents.create_version(
    agent_name="bing-search-agent",
    definition=PromptAgentDefinition(
        model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
        instructions="You are a helpful assistant with web search capabilities.",
        tools=[
            BingGroundingAgentTool(
                bing_grounding=BingGroundingSearchToolParameters(
                    search_configurations=[
                        BingGroundingSearchConfiguration(
                            project_connection_id=bing_connection.id
                        )
                    ]
                )
            )
        ],
    ),
)
```

## AzureAISearchAgentTool

Enterprise search over your Azure AI Search indexes.

```python
from azure.ai.projects.models import (
    AzureAISearchAgentTool,
    AzureAISearchToolResource,
    AISearchIndexResource,
    AzureAISearchQueryType,
    PromptAgentDefinition,
)

# Get search connection
search_connection = project_client.connections.get(
    os.environ["AI_SEARCH_PROJECT_CONNECTION_NAME"]
)

agent = project_client.agents.create_version(
    agent_name="enterprise-search-agent",
    definition=PromptAgentDefinition(
        model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
        instructions="""You are a helpful assistant. Always provide citations 
        using format: [message_idx:search_idx source].""",
        tools=[
            AzureAISearchAgentTool(
                azure_ai_search=AzureAISearchToolResource(
                    indexes=[
                        AISearchIndexResource(
                            project_connection_id=search_connection.id,
                            index_name=os.environ["AI_SEARCH_INDEX_NAME"],
                            query_type=AzureAISearchQueryType.SIMPLE,
                        ),
                    ]
                )
            )
        ],
    ),
)
```

### Query Types

```python
from azure.ai.projects.models import AzureAISearchQueryType

# Available query types:
# - AzureAISearchQueryType.SIMPLE: Simple keyword search
# - AzureAISearchQueryType.SEMANTIC: Semantic ranking
# - AzureAISearchQueryType.VECTOR: Vector search
# - AzureAISearchQueryType.VECTOR_SIMPLE_HYBRID: Vector + keyword hybrid
# - AzureAISearchQueryType.VECTOR_SEMANTIC_HYBRID: Vector + semantic hybrid
```

## OpenApiTool

Call external REST APIs defined by OpenAPI spec.

```python
from azure.ai.agents.models import OpenApiTool, OpenApiAnonymousAuthDetails

openapi_spec = """
openapi: 3.0.0
info:
  title: Weather API
  version: 1.0.0
paths:
  /weather:
    get:
      summary: Get weather
      parameters:
        - name: location
          in: query
          required: true
          schema:
            type: string
      responses:
        '200':
          description: Weather data
"""

openapi_tool = OpenApiTool(
    name="weather_api",
    spec=openapi_spec,
    description="Get weather information",
    auth=OpenApiAnonymousAuthDetails(),
)

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="api-agent",
    instructions="Use the weather API to get weather data.",
    tools=openapi_tool.definitions,
)
```

## McpTool

Model Context Protocol server integration.

```python
from azure.ai.agents.models import McpTool

mcp_tool = McpTool(
    server_label="my-mcp-server",
    server_url="http://localhost:3000",
    allowed_tools=["search", "calculate"],
)

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="mcp-agent",
    instructions="Use MCP tools for specialized operations.",
    tools=mcp_tool.definitions,
)
```

## SharepointTool

Search SharePoint content.

```python
from azure.ai.agents.models import SharepointTool

sharepoint = SharepointTool(connection_id=os.environ["SHAREPOINT_CONNECTION_ID"])

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="sharepoint-agent",
    instructions="Search SharePoint for documents.",
    tools=sharepoint.definitions,
)
```

## ConnectedAgentTool

Multi-agent orchestration.

```python
from azure.ai.agents.models import ConnectedAgentTool

# Connect to another agent
connected_agent = ConnectedAgentTool(
    agent_id=other_agent.id,
    name="specialist-agent",
    description="A specialist agent for complex queries",
)

orchestrator = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="orchestrator",
    instructions="Delegate complex tasks to the specialist agent.",
    tools=connected_agent.definitions,
)
```

## ToolSet Pattern

Combine multiple tools:

```python
from azure.ai.agents.models import ToolSet, FunctionTool, CodeInterpreterTool

def my_function(x: int) -> int:
    """Double a number."""
    return x * 2

toolset = ToolSet()
toolset.add(FunctionTool(functions=[my_function]))
toolset.add(CodeInterpreterTool())

# Enable auto function calls
project_client.agents.enable_auto_function_calls(toolset)

agent = project_client.agents.create_agent(
    model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
    name="multi-tool-agent",
    instructions="You have multiple tools available.",
    toolset=toolset,
)

# Pass toolset to run for auto-execution
run = project_client.agents.runs.create_and_process(
    thread_id=thread.id,
    agent_id=agent.id,
    toolset=toolset,
)
```

## Tools Quick Reference

| Tool | Class | Connection Required | Use Case |
|------|-------|---------------------|----------|
| Code Interpreter | `CodeInterpreterTool` | No | Execute Python, generate files |
| File Search | `FileSearchTool` | No | RAG over uploaded documents |
| Function | `FunctionTool` | No | Call custom Python functions |
| Bing Grounding | `BingGroundingTool` | Yes | Web search |
| Azure AI Search | `AzureAISearchAgentTool` | Yes | Enterprise search |
| OpenAPI | `OpenApiTool` | No | Call REST APIs |
| MCP | `McpTool` | No | MCP server integration |
| SharePoint | `SharepointTool` | Yes | SharePoint search |
| Fabric | `FabricTool` | Yes | Microsoft Fabric integration |
| Connected Agent | `ConnectedAgentTool` | No | Multi-agent orchestration |

#!/usr/bin/env python3
"""
Batch Evaluation CLI Tool

Run batch evaluations on test datasets using Azure AI Projects SDK.
Supports quality, safety, agent evaluators, and OpenAI graders.

Usage:
    python run_batch_evaluation.py --data test_data.jsonl --evaluators coherence relevance
    python run_batch_evaluation.py --data test_data.jsonl --evaluators coherence --output results.json
    python run_batch_evaluation.py --data test_data.jsonl --safety
    python run_batch_evaluation.py --data test_data.jsonl --agent --evaluators intent_resolution task_adherence

Environment Variables:
    AZURE_AI_PROJECT_ENDPOINT     - Azure AI project endpoint (required)
    AZURE_AI_MODEL_DEPLOYMENT_NAME - Model deployment name (default: gpt-4o-mini)
"""

import argparse
import json
import os
import sys
import time
from pathlib import Path
from typing import Any

from azure.ai.projects import AIProjectClient
from azure.identity import DefaultAzureCredential
from openai.types.evals.create_eval_jsonl_run_data_source_param import (
    CreateEvalJSONLRunDataSourceParam,
    SourceFileContent,
    SourceFileContentContent,
)
from openai.types.eval_create_params import DataSourceConfigCustom


# Built-in evaluators by category
QUALITY_EVALUATORS = [
    "coherence",
    "relevance",
    "fluency",
    "groundedness",
]
SAFETY_EVALUATORS = [
    "violence",
    "sexual",
    "self_harm",
    "hate_unfairness",
]
AGENT_EVALUATORS = [
    "intent_resolution",
    "response_completeness",
    "task_adherence",
    "tool_call_accuracy",
]
NLP_EVALUATORS = ["f1", "rouge", "bleu", "gleu", "meteor"]


def load_jsonl(path: str) -> list[dict]:
    """Load JSONL file into list of dicts."""
    data = []
    with open(path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                data.append(json.loads(line))
    return data


def build_data_source(
    data: list[dict],
    is_agent: bool = False,
) -> CreateEvalJSONLRunDataSourceParam:
    """Build data source from loaded data."""
    content = []
    for item in data:
        if is_agent:
            # Agent data: extract sample fields from item
            sample = {
                "output_text": item.pop("output_text", item.get("response", "")),
            }
            if "output_items" in item:
                sample["output_items"] = item.pop("output_items")
            content.append(SourceFileContentContent(item=item, sample=sample))
        else:
            content.append(SourceFileContentContent(item=item, sample={}))

    return CreateEvalJSONLRunDataSourceParam(
        type="jsonl",
        source=SourceFileContent(type="file_content", content=content),
    )


def build_data_source_config(
    data: list[dict],
    is_agent: bool = False,
) -> DataSourceConfigCustom:
    """Build data source config based on data schema."""
    # Infer schema from first item
    if not data:
        raise ValueError("Data is empty")

    first_item = data[0]
    properties = {}
    required = []

    for key in first_item:
        if key not in ["output_text", "output_items"]:  # Agent fields go in sample
            properties[key] = {"type": "string"}
            required.append(key)

    return DataSourceConfigCustom(
        type="custom",
        item_schema={
            "type": "object",
            "properties": properties,
            "required": required,
        },
        include_sample_schema=is_agent,
    )


def build_testing_criteria(
    evaluator_names: list[str],
    deployment_name: str,
    is_agent: bool = False,
) -> list[dict]:
    """Build testing criteria for the specified evaluators."""
    criteria = []

    for name in evaluator_names:
        # Determine data mapping based on evaluator type
        if name in QUALITY_EVALUATORS:
            if name == "groundedness":
                data_mapping = {
                    "query": "{{item.query}}",
                    "context": "{{item.context}}",
                    "response": "{{item.response}}",
                }
            else:
                data_mapping = {
                    "query": "{{item.query}}",
                    "response": "{{item.response}}",
                }
            needs_model = True

        elif name in SAFETY_EVALUATORS:
            data_mapping = {
                "query": "{{item.query}}",
                "response": "{{item.response}}",
            }
            needs_model = False  # Safety evaluators may not need deployment

        elif name in AGENT_EVALUATORS:
            if is_agent:
                if name == "tool_call_accuracy":
                    data_mapping = {
                        "query": "{{item.query}}",
                        "response": "{{sample.output_items}}",
                    }
                else:
                    data_mapping = {
                        "query": "{{item.query}}",
                        "response": "{{sample.output_text}}",
                    }
            else:
                data_mapping = {
                    "query": "{{item.query}}",
                    "response": "{{item.response}}",
                }
            needs_model = True

        elif name in NLP_EVALUATORS:
            data_mapping = {
                "response": "{{item.response}}",
                "ground_truth": "{{item.ground_truth}}",
            }
            needs_model = False

        else:
            print(f"Warning: Unknown evaluator '{name}', skipping")
            continue

        criterion = {
            "type": "azure_ai_evaluator",
            "name": name,
            "evaluator_name": f"builtin.{name}",
            "data_mapping": data_mapping,
        }

        if needs_model:
            criterion["initialization_parameters"] = {"deployment_name": deployment_name}

        criteria.append(criterion)

    return criteria


def run_evaluation(
    endpoint: str,
    data_path: str,
    evaluator_names: list[str],
    deployment_name: str,
    is_agent: bool = False,
) -> dict[str, Any]:
    """Run batch evaluation using Azure AI Projects SDK."""
    # Load data
    data = load_jsonl(data_path)
    print(f"Loaded {len(data)} items from {data_path}")

    # Build data source and config
    data_source = build_data_source(data, is_agent=is_agent)
    data_source_config = build_data_source_config(data, is_agent=is_agent)

    # Build testing criteria
    testing_criteria = build_testing_criteria(
        evaluator_names,
        deployment_name,
        is_agent=is_agent,
    )

    if not testing_criteria:
        raise ValueError("No valid testing criteria configured")

    print(f"Configured {len(testing_criteria)} evaluators")

    # Create client and run evaluation
    with (
        DefaultAzureCredential() as credential,
        AIProjectClient(endpoint=endpoint, credential=credential) as project_client,
    ):
        openai_client = project_client.get_openai_client()

        # Create evaluation definition
        eval_object = openai_client.evals.create(
            name=f"Batch Evaluation - {Path(data_path).stem}",
            data_source_config=data_source_config,
            testing_criteria=testing_criteria,
        )
        print(f"Created evaluation: {eval_object.id}")

        # Create and run evaluation
        run = openai_client.evals.runs.create(
            eval_id=eval_object.id,
            name="CLI Run",
            data_source=data_source,
        )
        print(f"Started run: {run.id}")

        # Poll for completion
        while run.status not in ["completed", "failed", "cancelled"]:
            print(f"Status: {run.status}...")
            time.sleep(5)
            run = openai_client.evals.runs.retrieve(
                eval_id=eval_object.id,
                run_id=run.id,
            )

        if run.status != "completed":
            raise RuntimeError(f"Evaluation run {run.status}: {getattr(run, 'error', 'Unknown error')}")

        print(f"Run completed: {run.status}")

        # Retrieve results
        output_items = list(
            openai_client.evals.runs.output_items.list(
                eval_id=eval_object.id,
                run_id=run.id,
            )
        )

        # Aggregate metrics
        metrics: dict[str, list[float]] = {}
        rows = []

        for output_item in output_items:
            row_results = {}
            for result in output_item.results:
                if result.score is not None:
                    if result.name not in metrics:
                        metrics[result.name] = []
                    metrics[result.name].append(result.score)
                    row_results[result.name] = result.score
            rows.append(row_results)

        # Calculate averages
        avg_metrics = {}
        for name, scores in metrics.items():
            avg_metrics[name] = sum(scores) / len(scores) if scores else 0.0

        return {
            "eval_id": eval_object.id,
            "run_id": run.id,
            "status": run.status,
            "metrics": avg_metrics,
            "rows": rows,
            "total_items": len(output_items),
        }


def main():
    parser = argparse.ArgumentParser(
        description="Run batch evaluation on test datasets using Azure AI Projects SDK",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )

    parser.add_argument("--data", "-d", required=True, help="Path to JSONL data file")
    parser.add_argument(
        "--evaluators",
        "-e",
        nargs="+",
        default=["coherence", "relevance"],
        help=f"Evaluators to run. Quality: {QUALITY_EVALUATORS}, "
        f"Safety: {SAFETY_EVALUATORS}, Agent: {AGENT_EVALUATORS}, NLP: {NLP_EVALUATORS}",
    )
    parser.add_argument(
        "--safety",
        action="store_true",
        help="Include all safety evaluators",
    )
    parser.add_argument(
        "--agent",
        action="store_true",
        help="Include all agent evaluators (uses sample.output_text for response)",
    )
    parser.add_argument(
        "--output",
        "-o",
        help="Output file for results (JSON)",
    )
    parser.add_argument(
        "--deployment",
        default=None,
        help="Model deployment name (overrides AZURE_AI_MODEL_DEPLOYMENT_NAME)",
    )

    args = parser.parse_args()

    # Validate environment
    endpoint = os.environ.get("AZURE_AI_PROJECT_ENDPOINT")
    if not endpoint:
        print("Error: AZURE_AI_PROJECT_ENDPOINT environment variable required")
        sys.exit(1)

    deployment = args.deployment or os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "gpt-4o-mini")

    # Validate data file
    data_path = Path(args.data)
    if not data_path.exists():
        print(f"Error: Data file not found: {args.data}")
        sys.exit(1)

    # Build evaluator list
    evaluator_names = list(args.evaluators)
    if args.safety:
        evaluator_names.extend(SAFETY_EVALUATORS)
    if args.agent:
        evaluator_names.extend(AGENT_EVALUATORS)

    # Remove duplicates while preserving order
    seen = set()
    unique_evaluators = []
    for e in evaluator_names:
        if e not in seen:
            seen.add(e)
            unique_evaluators.append(e)
    evaluator_names = unique_evaluators

    print(f"Running evaluation with: {evaluator_names}")
    print(f"Data file: {args.data}")
    print(f"Deployment: {deployment}")
    print(f"Agent mode: {args.agent}")

    # Run evaluation
    try:
        result = run_evaluation(
            endpoint=endpoint,
            data_path=str(data_path),
            evaluator_names=evaluator_names,
            deployment_name=deployment,
            is_agent=args.agent,
        )
    except Exception as e:
        print(f"Error during evaluation: {e}")
        sys.exit(1)

    # Output results
    print("\n=== Evaluation Results ===")
    print(f"Eval ID: {result['eval_id']}")
    print(f"Run ID: {result['run_id']}")
    print(f"Status: {result['status']}")
    print(f"Total Items: {result['total_items']}")
    print("\nMetrics:")
    for metric, value in sorted(result["metrics"].items()):
        print(f"  {metric}: {value:.4f}")

    # Save to file if requested
    if args.output:
        output_path = Path(args.output)
        with open(output_path, "w", encoding="utf-8") as f:
            json.dump(result, f, indent=2, default=str)
        print(f"\nResults saved to: {args.output}")

    print("\nEvaluation complete!")


if __name__ == "__main__":
    main()