Tool Use Mastery -- המיומנות הכי חשובה של סוכן AI

# ---- BAD DESCRIPTIONS ----

# Bad #1: Too vague
bad_tool_1 = {
    "name": "search",
    "description": "Search for things."
}
# What things? Where? When to use it vs ask directly?

# Bad #2: Too short, no context
bad_tool_2 = {
    "name": "get_user",
    "description": "Gets a user."
}
# By what? ID? Email? Name? What does it return?

# Bad #3: Implementation details, not usage guidance
bad_tool_3 = {
    "name": "send_email",
    "description": "Uses SMTP to connect to mail server and send."
}
# The model doesn't care about SMTP. It needs to know WHEN to send.


# ---- GOOD DESCRIPTIONS ----

# Good #1: Clear, specific, includes when to use
good_tool_1 = {
    "name": "search_knowledge_base",
    "description": """Search the company's internal knowledge base for answers
to questions about products, policies, and procedures.

Use this tool when:
- The user asks about company products or services
- The user needs information about internal policies
- You need to verify facts before answering

Do NOT use this tool when:
- The user is asking a general knowledge question (answer directly)
- The user is asking about competitors (you don't have that data)
- The user wants real-time information (knowledge base is updated weekly)

Returns: A list of relevant documents with titles and snippets."""
}

# Good #2: Specific about inputs and outputs
good_tool_2 = {
    "name": "get_user_profile",
    "description": """Retrieve a user's profile by their email address.

Returns the user's name, role, department, and join date.
Returns an error if the email is not found in the system.

Use this before personalizing responses or checking permissions.
Do NOT use this for authentication -- use verify_credentials instead."""
}

# Good #3: Includes edge cases
good_tool_3 = {
    "name": "send_notification",
    "description": """Send a notification to a user via their preferred channel
(email, Slack, or SMS).

IMPORTANT: Always confirm with the user before sending.
The notification is sent immediately and CANNOT be recalled.

Use this when the user explicitly asks to notify someone.
Do NOT use this for system-generated alerts -- use log_event instead.

If the recipient has no preferred channel set, defaults to email."""
}

import { tool } from 'ai';
import { z } from 'zod';

// Without negative instructions: model confuses these two
const searchProducts = tool({
  description: `Search the product catalog by keyword.
Use this when the user wants to FIND products.
Do NOT use this to check inventory -- use check_inventory instead.
Do NOT use this for order history -- use get_orders instead.`,
  parameters: z.object({
    query: z.string().describe('Search keywords, e.g. "red shoes size 42"'),
    category: z.string().optional().describe('Filter by category'),
  }),
  execute: async ({ query, category }) => {
    // ... search logic
  },
});

const checkInventory = tool({
  description: `Check if a specific product is in stock and available for purchase.
Use this AFTER the user has identified a specific product (by ID or exact name).
Do NOT use this to search for products -- use search_products instead.
Returns: { in_stock: boolean, quantity: number, warehouse: string }`,
  parameters: z.object({
    product_id: z.string().describe('The product ID (e.g. "PROD-12345")'),
  }),
  execute: async ({ product_id }) => {
    // ... inventory check logic
  },
});

from anthropic import Anthropic

client = Anthropic()

# ---- RULE 1: Use enums when possible ----
# BAD: status as free string
bad_schema = {
    "type": "object",
    "properties": {
        "status": {"type": "string"}  # Model might send "Open", "OPEN", "opened"
    }
}

# GOOD: status as enum
good_schema = {
    "type": "object",
    "properties": {
        "status": {
            "type": "string",
            "enum": ["open", "closed", "pending"],  # Exact values only
            "description": "The ticket status to filter by"
        }
    },
    "required": ["status"]
}


# ---- RULE 2: Make descriptions specific ----
# BAD
bad_param = {"type": "string", "description": "The query"}

# GOOD
good_param = {
    "type": "string",
    "description": "Search query in natural language, e.g. 'red shoes size 42'. Max 200 characters."
}


# ---- RULE 3: Keep it flat -- avoid nested objects ----
# BAD: deeply nested
bad_nested = {
    "type": "object",
    "properties": {
        "filter": {
            "type": "object",
            "properties": {
                "price": {
                    "type": "object",
                    "properties": {
                        "min": {"type": "number"},
                        "max": {"type": "number"}
                    }
                }
            }
        }
    }
}

# GOOD: flat
good_flat = {
    "type": "object",
    "properties": {
        "min_price": {
            "type": "number",
            "description": "Minimum price in USD. Default: 0"
        },
        "max_price": {
            "type": "number",
            "description": "Maximum price in USD. Default: no limit"
        }
    }
}


# ---- RULE 4: Make everything required unless optional ----
# If a parameter is needed 90% of the time, make it required.
# Optional params should have clear defaults documented.


# ---- RULE 5: Validate on the server side ----
# NEVER trust the model's output -- always validate.
def execute_tool(params: dict) -> str:
    # Validate required fields
    if "query" not in params:
        return "Error: 'query' is required"

    # Validate types
    if not isinstance(params["query"], str):
        return "Error: 'query' must be a string"

    # Validate constraints
    if len(params["query"]) > 200:
        return "Error: 'query' must be 200 characters or less"

    # Validate enum values
    if params.get("status") and params["status"] not in ["open", "closed", "pending"]:
        return f"Error: 'status' must be one of: open, closed, pending"

    # ... actual logic
    return "Success"


# ---- RULE 6: Fewer params = better accuracy ----
# BAD: 8 parameters, most optional
bad_tool = {
    "name": "search",
    "input_schema": {
        "type": "object",
        "properties": {
            "query": {"type": "string"},
            "category": {"type": "string"},
            "min_price": {"type": "number"},
            "max_price": {"type": "number"},
            "color": {"type": "string"},
            "size": {"type": "string"},
            "brand": {"type": "string"},
            "sort_by": {"type": "string"}
        },
        "required": ["query"]
    }
}

# GOOD: 2 parameters, focused
good_tool = {
    "name": "search_products",
    "description": "Search products. Filters can be included in the query naturally.",
    "input_schema": {
        "type": "object",
        "properties": {
            "query": {
                "type": "string",
                "description": "Natural language search, e.g. 'red Nike shoes under $100 size 42'"
            },
            "max_results": {
                "type": "integer",
                "description": "Max results to return (1-50). Default: 10"
            }
        },
        "required": ["query"]
    }
}

import json

# ---- Strategy 1: Return only relevant fields ----
def search_users_bad(query: str) -> str:
    """Returns entire user objects -- wasteful."""
    users = db.search(query)
    return json.dumps(users)  # Each user has 50 fields!

def search_users_good(query: str) -> str:
    """Returns only what the model needs."""
    users = db.search(query)
    return json.dumps([
        {"id": u["id"], "name": u["name"], "email": u["email"]}
        for u in users
    ])


# ---- Strategy 2: Truncate with summary ----
def read_document(doc_id: str) -> str:
    """Read a document, truncating if too long."""
    doc = db.get_document(doc_id)
    content = doc["content"]

    MAX_CHARS = 4000  # ~1,000 tokens

    if len(content) <= MAX_CHARS:
        return content

    # Truncate and add a note
    return (
        content[:MAX_CHARS]
        + f"\n\n[TRUNCATED -- document is {len(content)} chars. "
        + "Use read_document_section(doc_id, page) for specific pages.]"
    )


# ---- Strategy 3: Paginate ----
def list_orders(user_id: str, page: int = 1, page_size: int = 10) -> str:
    """List orders with pagination."""
    orders = db.get_orders(user_id)
    total = len(orders)
    start = (page - 1) * page_size
    end = start + page_size

    return json.dumps({
        "orders": [
            {"id": o["id"], "date": o["date"], "total": o["total"], "status": o["status"]}
            for o in orders[start:end]
        ],
        "page": page,
        "total_pages": (total + page_size - 1) // page_size,
        "total_orders": total
    })


# ---- Strategy 4: Structured error messages ----
def get_user(email: str) -> str:
    """Return helpful errors the model can act on."""
    user = db.find_by_email(email)
    if not user:
        return json.dumps({
            "error": "user_not_found",
            "message": f"No user with email '{email}'. Check spelling or ask user to confirm.",
            "suggestion": "Try search_users tool with a name instead."
        })
    return json.dumps({
        "id": user["id"],
        "name": user["name"],
        "email": user["email"],
        "role": user["role"]
    })

import { generateText, tool, stepCountIs } from 'ai';
import { anthropic } from '@ai-sdk/anthropic';
import { experimental_createMCPClient as createMCPClient } from 'ai';
import { z } from 'zod';

// Native tool: core business logic (fast, internal)
const calculateDiscount = tool({
  description: `Calculate the discount for a customer based on their tier
and purchase history. Internal business logic -- fast, no external calls.`,
  parameters: z.object({
    customer_id: z.string(),
    order_total: z.number(),
  }),
  execute: async ({ customer_id, order_total }) => {
    const tier = await getCustomerTier(customer_id);
    const discount = tier === 'gold' ? 0.15 : tier === 'silver' ? 0.10 : 0.05;
    return { discount_percent: discount * 100, final_total: order_total * (1 - discount) };
  },
});

// MCP tool: external integration (Slack notifications)
const mcpClient = await createMCPClient({
  transport: { type: 'stdio', command: 'npx', args: ['-y', '@anthropic/mcp-slack'] },
});
const slackTools = await mcpClient.tools();

// Combine both in one agent
const { text } = await generateText({
  model: anthropic('claude-sonnet-4-20250514'),
  tools: {
    calculateDiscount,       // Native: fast, internal
    ...slackTools,            // MCP: Slack integration
  },
  stopWhen: stepCountIs(5),
  system: `You are an order processing agent. Use calculateDiscount for
pricing logic. Use Slack tools to notify the sales team of large orders.`,
  prompt: userMessage,
});

from anthropic import Anthropic

client = Anthropic()

# Tool registry -- all available tools
TOOL_REGISTRY = {
    # Available to everyone
    "search_products": {
        "name": "search_products",
        "description": "Search the product catalog. Available to all users.",
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string", "description": "Search query"}
            },
            "required": ["query"]
        },
        "roles": ["user", "support", "admin"]
    },
    "get_order_status": {
        "name": "get_order_status",
        "description": "Check the status of an order by order ID.",
        "input_schema": {
            "type": "object",
            "properties": {
                "order_id": {"type": "string", "description": "Order ID (ORD-XXXX)"}
            },
            "required": ["order_id"]
        },
        "roles": ["user", "support", "admin"]
    },
    # Support only
    "issue_refund": {
        "name": "issue_refund",
        "description": """Issue a refund for an order. REQUIRES confirmation before executing.
Use only after verifying the order exists and the refund reason is valid.
Do NOT use for orders older than 30 days -- escalate to admin instead.""",
        "input_schema": {
            "type": "object",
            "properties": {
                "order_id": {"type": "string"},
                "amount": {"type": "number", "description": "Refund amount in USD"},
                "reason": {
                    "type": "string",
                    "enum": ["defective", "wrong_item", "late_delivery", "customer_request"]
                }
            },
            "required": ["order_id", "amount", "reason"]
        },
        "roles": ["support", "admin"]
    },
    # Admin only
    "delete_user": {
        "name": "delete_user",
        "description": """Permanently delete a user account. THIS CANNOT BE UNDONE.
Only use when explicitly instructed by an admin after identity verification.
Returns the deleted user's data for backup purposes.""",
        "input_schema": {
            "type": "object",
            "properties": {
                "user_id": {"type": "string"},
                "confirmation": {
                    "type": "string",
                    "description": "Must be 'CONFIRM_DELETE_'"
                }
            },
            "required": ["user_id", "confirmation"]
        },
        "roles": ["admin"]
    }
}


def get_tools_for_role(role: str) -> list[dict]:
    """Return only tools that are available for this role."""
    return [
        {k: v for k, v in tool_def.items() if k != "roles"}
        for tool_def in TOOL_REGISTRY.values()
        if role in tool_def["roles"]
    ]


def run_agent(user_message: str, user_role: str = "user"):
    """Run agent with role-appropriate tools."""
    tools = get_tools_for_role(user_role)

    print(f"[{user_role}] has access to {len(tools)} tools: "
          f"{[t['name'] for t in tools]}")

    response = client.messages.create(
        model="claude-sonnet-4-20250514",
        max_tokens=1024,
        system=f"You are a customer service agent. User role: {user_role}.",
        tools=tools,
        messages=[{"role": "user", "content": user_message}]
    )
    return response


# Demo
run_agent("Delete user 12345", user_role="user")
# [user] has access to 2 tools: ['search_products', 'get_order_status']
# Agent can't delete -- doesn't even see the tool!

run_agent("Delete user 12345", user_role="admin")
# [admin] has access to 4 tools: ['search_products', 'get_order_status',
#   'issue_refund', 'delete_user']
# Agent CAN delete -- but still requires confirmation

import { generateText, tool, stepCountIs, ModelMessage } from 'ai';
import { anthropic } from '@ai-sdk/anthropic';
import { z } from 'zod';

// Phase 1 tools: gathering information
const phase1Tools = {
  search_products: tool({
    description: 'Search the product catalog. Use in the research phase.',
    parameters: z.object({ query: z.string() }),
    execute: async ({ query }) => ({ results: [`Product A`, `Product B`] }),
  }),
  get_reviews: tool({
    description: 'Get reviews for a product. Use in the research phase.',
    parameters: z.object({ product_id: z.string() }),
    execute: async ({ product_id }) => ({ reviews: ['Great!', 'Good value'] }),
  }),
};

// Phase 2 tools: only available after user selects a product
const phase2Tools = {
  ...phase1Tools,
  add_to_cart: tool({
    description: 'Add a selected product to cart. Only use after user confirms.',
    parameters: z.object({
      product_id: z.string(),
      quantity: z.number().int().min(1).max(10),
    }),
    execute: async ({ product_id, quantity }) => ({ added: true }),
  }),
  apply_coupon: tool({
    description: 'Apply a coupon code to the cart.',
    parameters: z.object({ code: z.string() }),
    execute: async ({ code }) => ({ valid: true, discount: '10%' }),
  }),
};

// Phase 3 tools: checkout (only after cart has items)
const phase3Tools = {
  ...phase2Tools,
  checkout: tool({
    description: 'Process checkout. Only after cart review and user confirmation.',
    parameters: z.object({
      payment_method: z.enum(['credit_card', 'paypal', 'bank_transfer']),
    }),
    execute: async ({ payment_method }) => ({ order_id: 'ORD-789' }),
  }),
};

// Agent with progressive tool loading
async function shoppingAgent(messages: ModelMessage[]) {
  const phase = detectPhase(messages);
  const tools = phase === 1 ? phase1Tools
              : phase === 2 ? phase2Tools
              : phase3Tools;

  console.log(`Phase ${phase}: ${Object.keys(tools).length} tools available`);

  return generateText({
    model: anthropic('claude-sonnet-4-20250514'),
    tools,
    stopWhen: stepCountIs(5),
    messages,
  });
}

function detectPhase(messages: ModelMessage[]): number {
  const text = messages.map(m => String(m.content)).join(' ').toLowerCase();
  if (text.includes('checkout') || text.includes('buy now')) return 3;
  if (text.includes('add to cart') || text.includes('i want this')) return 2;
  return 1;
}

from anthropic import Anthropic
import json

client = Anthropic()

# Tools that naturally compose sequentially
tools = [
    {
        "name": "search_customer",
        "description": "Find a customer by name or email. Returns customer ID and details.",
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {"type": "string", "description": "Name or email to search"}
            },
            "required": ["query"]
        }
    },
    {
        "name": "get_orders",
        "description": """Get orders for a specific customer by their ID.
Use search_customer first to get the customer ID.
Returns list of orders with status, amount, and date.""",
        "input_schema": {
            "type": "object",
            "properties": {
                "customer_id": {"type": "string", "description": "Customer ID from search_customer"}
            },
            "required": ["customer_id"]
        }
    },
    {
        "name": "issue_refund",
        "description": """Issue a refund for a specific order.
Use get_orders first to find the order ID.
Requires confirmation before executing.""",
        "input_schema": {
            "type": "object",
            "properties": {
                "order_id": {"type": "string"},
                "amount": {"type": "number"},
                "reason": {"type": "string", "enum": ["defective", "wrong_item", "late"]}
            },
            "required": ["order_id", "amount", "reason"]
        }
    }
]

# The model will chain: search_customer -> get_orders -> issue_refund
# Each step uses the output of the previous step
# "Refund Yael's last order because it was defective"
# Step 1: search_customer("Yael") -> { customer_id: "C-123" }
# Step 2: get_orders("C-123") -> [{ order_id: "ORD-456", amount: 89.99 }]
# Step 3: issue_refund("ORD-456", 89.99, "defective") -> { success: true }

import { generateText, tool, stepCountIs } from 'ai';
import { anthropic } from '@ai-sdk/anthropic';
import { z } from 'zod';

// These tools can run in parallel -- no dependencies between them
const tools = {
  get_weather: tool({
    description: 'Get current weather for a city.',
    parameters: z.object({ city: z.string() }),
    execute: async ({ city }) => ({ city, temp: 28, condition: 'sunny' }),
  }),
  get_flights: tool({
    description: 'Search flights between two cities.',
    parameters: z.object({
      from: z.string(),
      to: z.string(),
      date: z.string(),
    }),
    execute: async ({ from, to, date }) => ({
      flights: [
        { airline: 'El Al', price: 450, departure: '08:00' },
        { airline: 'Lufthansa', price: 380, departure: '14:30' },
      ]
    }),
  }),
  get_hotels: tool({
    description: 'Search hotels in a city.',
    parameters: z.object({
      city: z.string(),
      checkin: z.string(),
      checkout: z.string(),
    }),
    execute: async ({ city, checkin, checkout }) => ({
      hotels: [
        { name: 'Grand Hotel', price: 120, rating: 4.5 },
        { name: 'Budget Inn', price: 65, rating: 3.8 },
      ]
    }),
  }),
};

// When user asks "Plan a trip to Berlin next week":
// The model will call get_weather, get_flights, AND get_hotels in parallel
// (Anthropic/OpenAI support parallel tool calls natively)
const { text } = await generateText({
  model: anthropic('claude-sonnet-4-20250514'),
  tools,
  stopWhen: stepCountIs(3),
  prompt: 'Plan a trip from Tel Aviv to Berlin next Monday for 4 nights.',
});

import time
import json
from typing import Any
from functools import wraps


# ---- Strategy 1: Retry with backoff ----
def retry_tool(max_retries: int = 3, backoff_factor: float = 1.0):
    """Decorator that retries a tool function on failure."""
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            last_error = None
            for attempt in range(max_retries):
                try:
                    return func(*args, **kwargs)
                except Exception as e:
                    last_error = e
                    if attempt < max_retries - 1:
                        wait = backoff_factor * (2 ** attempt)
                        time.sleep(wait)
            return json.dumps({
                "error": "tool_failed_after_retries",
                "message": f"Failed after {max_retries} attempts: {str(last_error)}",
                "suggestion": "Try again later or use an alternative approach."
            })
        return wrapper
    return decorator


@retry_tool(max_retries=3, backoff_factor=0.5)
def web_search(query: str) -> str:
    """Search the web -- with automatic retries."""
    results = search_api.search(query)  # might fail
    return json.dumps(results)


# ---- Strategy 2: Fallback tools ----
def search_with_fallback(query: str) -> str:
    """Try primary search, fall back to cache if it fails."""
    try:
        # Primary: live web search
        results = web_search_api.search(query)
        return json.dumps({"source": "live", "results": results})
    except Exception:
        try:
            # Fallback 1: cached results
            cached = cache.get(f"search:{query}")
            if cached:
                return json.dumps({
                    "source": "cache",
                    "results": cached,
                    "warning": "These results may be outdated (from cache)."
                })
        except Exception:
            pass

        # Fallback 2: graceful degradation
        return json.dumps({
            "error": "search_unavailable",
            "message": "Web search is temporarily unavailable.",
            "suggestion": "I can try to answer based on my training data, "
                         "but I may not have the latest information."
        })


# ---- Strategy 3: Timeout protection ----
import concurrent.futures

def tool_with_timeout(func, timeout_seconds: int = 10):
    """Run a tool with a timeout."""
    def wrapper(*args, **kwargs):
        with concurrent.futures.ThreadPoolExecutor() as executor:
            future = executor.submit(func, *args, **kwargs)
            try:
                return future.result(timeout=timeout_seconds)
            except concurrent.futures.TimeoutError:
                return json.dumps({
                    "error": "timeout",
                    "message": f"Tool did not respond within {timeout_seconds}s.",
                    "suggestion": "The service may be slow. Try with a simpler query."
                })
    return wrapper


# ---- Strategy 4: Helpful error messages ----
# BAD: errors the model can't act on
bad_error = '{"error": "500 Internal Server Error"}'
bad_error_2 = '{"error": "NullPointerException at line 42"}'

# GOOD: errors the model CAN act on
good_error = json.dumps({
    "error": "rate_limit_exceeded",
    "message": "API rate limit reached (100 requests/minute).",
    "retry_after_seconds": 30,
    "suggestion": "Wait 30 seconds before trying again, or reduce the scope of the query."
})

import { tool } from 'ai';
import { z } from 'zod';

// Tool with built-in confirmation
const sendEmail = tool({
  description: `Send an email. This is IRREVERSIBLE -- the email is sent immediately.
ALWAYS preview the email content with the user before calling this tool.
Do NOT call this tool until the user has explicitly confirmed "yes, send it".`,
  parameters: z.object({
    to: z.string().email().describe('Recipient email address'),
    subject: z.string().max(200).describe('Email subject line'),
    body: z.string().describe('Email body (plain text)'),
    confirmed: z.boolean().describe(
      'Must be true. Set to true only after user explicitly confirmed.'
    ),
  }),
  execute: async ({ to, subject, body, confirmed }) => {
    // Server-side confirmation check
    if (!confirmed) {
      return {
        status: 'blocked',
        message: 'Cannot send without confirmation. Show the email to the user first.',
        preview: { to, subject, body_preview: body.slice(0, 200) }
      };
    }

    // Rate limiting
    const recentSends = await getRecentSendCount(to);
    if (recentSends > 5) {
      return {
        status: 'rate_limited',
        message: `Already sent ${recentSends} emails to ${to} today. Try again tomorrow.`
      };
    }

    // Actually send
    const result = await emailService.send({ to, subject, body });

    // Audit log
    await auditLog.write({
      action: 'email_sent',
      to, subject,
      timestamp: new Date(),
      agent_session: getCurrentSession()
    });

    return { status: 'sent', message_id: result.id };
  },
});

from mcp.server import FastMCP

app = FastMCP("secure-tools")

@app.tool(annotations={
    "readOnlyHint": True,         # This tool only reads data
    "destructiveHint": False,      # Doesn't change or delete anything
    "idempotentHint": True,        # Can be called multiple times safely
    "openWorldHint": False         # No external network calls
})
async def search_database(query: str) -> str:
    """Search the internal database. Read-only, safe to call repeatedly."""
    results = await db.search(query)
    return str(results)


@app.tool(annotations={
    "readOnlyHint": False,         # Modifies data
    "destructiveHint": True,       # Can delete or change things
    "idempotentHint": False,       # Multiple calls = multiple actions
    "openWorldHint": True          # Sends data externally
})
async def delete_record(record_id: str, confirmation: str) -> str:
    """Delete a record. DESTRUCTIVE and IRREVERSIBLE.
    confirmation must be 'DELETE_' to proceed."""
    expected = f"DELETE_{record_id}"
    if confirmation != expected:
        return f"Error: confirmation must be '{expected}'"
    await db.delete(record_id)
    return f"Record {record_id} deleted permanently."

import json
from dataclasses import dataclass
from anthropic import Anthropic

client = Anthropic()


@dataclass
class ToolTestCase:
    """A single test case for tool use evaluation."""
    user_message: str
    expected_tool: str              # Which tool should be called
    expected_params: dict           # Expected parameter values (partial match)
    description: str                # What this test case checks


# Define test cases
TEST_CASES = [
    ToolTestCase(
        user_message="What's the weather in Tel Aviv?",
        expected_tool="get_weather",
        expected_params={"city": "Tel Aviv"},
        description="Basic tool selection -- weather query"
    ),
    ToolTestCase(
        user_message="Find flights from TLV to Berlin on March 30",
        expected_tool="search_flights",
        expected_params={"from": "TLV", "to": "Berlin"},
        description="Parameter extraction -- cities and dates"
    ),
    ToolTestCase(
        user_message="What is 2+2?",
        expected_tool="__none__",  # Should NOT use any tool
        expected_params={},
        description="No-tool case -- simple question, answer directly"
    ),
    ToolTestCase(
        user_message="Send an email to yael@example.com about the meeting",
        expected_tool="send_email",
        expected_params={"to": "yael@example.com"},
        description="Destructive tool -- should also ask for confirmation"
    ),
]


def evaluate_tool_use(tools: list[dict], test_cases: list[ToolTestCase]) -> dict:
    """Run all test cases and compute metrics."""
    results = {
        "total": len(test_cases),
        "tool_selection_correct": 0,
        "param_accuracy_correct": 0,
        "details": []
    }

    for tc in test_cases:
        response = client.messages.create(
            model="claude-sonnet-4-20250514",
            max_tokens=1024,
            tools=tools,
            messages=[{"role": "user", "content": tc.user_message}]
        )

        # Check which tool was called
        tool_calls = [b for b in response.content if b.type == "tool_use"]

        if tc.expected_tool == "__none__":
            tool_correct = len(tool_calls) == 0
            param_correct = True
        elif len(tool_calls) > 0:
            actual_tool = tool_calls[0].name
            actual_params = tool_calls[0].input
            tool_correct = actual_tool == tc.expected_tool
            param_correct = all(
                actual_params.get(k) == v
                for k, v in tc.expected_params.items()
            )
        else:
            tool_correct = False
            param_correct = False

        if tool_correct:
            results["tool_selection_correct"] += 1
        if param_correct:
            results["param_accuracy_correct"] += 1

        results["details"].append({
            "test": tc.description,
            "expected_tool": tc.expected_tool,
            "actual_tool": tool_calls[0].name if tool_calls else "__none__",
            "tool_correct": tool_correct,
            "param_correct": param_correct,
        })

    # Compute percentages
    results["tool_selection_accuracy"] = (
        results["tool_selection_correct"] / results["total"] * 100
    )
    results["param_accuracy"] = (
        results["param_accuracy_correct"] / results["total"] * 100
    )

    return results


# Run evaluation
results = evaluate_tool_use(my_tools, TEST_CASES)
print(f"Tool Selection Accuracy: {results['tool_selection_accuracy']:.1f}%")
print(f"Parameter Accuracy: {results['param_accuracy']:.1f}%")

for detail in results["details"]:
    status = "PASS" if detail["tool_correct"] else "FAIL"
    print(f"  [{status}] {detail['test']}: "
          f"expected={detail['expected_tool']}, "
          f"actual={detail['actual_tool']}")

import { generateText, tool, stepCountIs } from 'ai';
import { anthropic } from '@ai-sdk/anthropic';
import { z } from 'zod';

// Create a tool that fails deliberately
let callCount = 0;
const flakyTool = tool({
  description: 'Get data from the database. May fail intermittently.',
  parameters: z.object({ query: z.string() }),
  execute: async ({ query }) => {
    callCount++;
    if (callCount <= 2) {
      // Fail first 2 times with helpful error
      return {
        error: 'connection_timeout',
        message: 'Database connection timed out.',
        suggestion: 'Try again -- the connection may recover.'
      };
    }
    // Succeed on 3rd try
    return { results: [{ id: 1, name: 'Test', value: 42 }] };
  },
});

// Test: does the agent retry and eventually succeed?
const { text, steps } = await generateText({
  model: anthropic('claude-sonnet-4-20250514'),
  tools: { query_db: flakyTool },
  stopWhen: stepCountIs(5),
  prompt: 'Look up the test data in the database.',
});

const toolCalls = steps.flatMap(s => s.toolCalls || []);
console.log(`Tool calls: ${toolCalls.length}`);     // Should be 3
console.log(`Final response: ${text}`);              // Should include the data
console.log(`Recovered: ${text.includes('42')}`);    // Should be true