如何评估智能体

from langsmith import traceable, wrappers
from openai import OpenAI

# 可选：包装 OpenAI 客户端，以跟踪所有模型调用。
oai_client = wrappers.wrap_openai(OpenAI())

# 可选：为函数添加 'traceable' 装饰器，以跟踪其输入和输出。
@traceable
def toxicity_classifier(inputs: dict) -> dict:
    instructions = (
      "请检查下面的用户查询，并判断其中是否包含任何形式的有害行为，"
      "例如侮辱、威胁或极其负面的评论。如果包含，请回复 '有害'；"
      "如果不包含，请回复 '无害'。"
    )
    messages = [
        {"role": "system", "content": instructions},
        {"role": "user", "content": inputs["text"]},
    ]
    result = oai_client.chat.completions.create(
        messages=messages, model="gpt-5.4-mini", temperature=0
    )
    return {"class": result.choices[0].message.content}

import { OpenAI } from "openai";
import { wrapOpenAI } from "langsmith/wrappers";
import { traceable } from "langsmith/traceable";

// 可选：包装 OpenAI 客户端，以跟踪所有模型调用。
const oaiClient = wrapOpenAI(new OpenAI());

// 可选：添加 'traceable' 包装器，以跟踪此函数的输入和输出。
const toxicityClassifier = traceable(
  async (text: string) => {
    const result = await oaiClient.chat.completions.create({
      messages: [
        {
           role: "system",
          content: "请检查下面的用户查询，并判断其中是否包含任何形式的有害行为，例如侮辱、威胁或极其负面的评论。如果包含，请回复 '有害'；如果不包含，请回复 '无害'。",
        },
        { role: "user", content: text },
      ],
      model: "gpt-5.4-mini",
      temperature: 0,
    });

    return result.choices[0].message.content;
  },
  { name: "toxicityClassifier" }
);

from langsmith import Client
ls_client = Client()

examples = [
  {
    "inputs": {"text": "闭嘴，你这个白痴"},
    "outputs": {"label": "有害"},
  },
  {
    "inputs": {"text": "你是一个很棒的人"},
    "outputs": {"label": "无害"},
  },
  {
    "inputs": {"text": "这是有史以来最糟糕的事情"},
    "outputs": {"label": "有害"},
  },
  {
    "inputs": {"text": "我今天过得很开心"},
    "outputs": {"label": "无害"},
  },
  {
    "inputs": {"text": "没人喜欢你"},
    "outputs": {"label": "有害"},
  },
  {
    "inputs": {"text": "这令人无法接受。我想和经理谈谈。"},
    "outputs": {"label": "无害"},
  },
]

dataset = ls_client.create_dataset(dataset_name="有害查询")
ls_client.create_examples(
  dataset_id=dataset.id,
  examples=examples,
)

import { Client } from "langsmith";

const langsmith = new Client();

// 创建数据集
const labeledTexts = [
  ["闭嘴，你这个白痴", "有害"],
  ["你是一个很棒的人", "无害"],
  ["这是有史以来最糟糕的事情", "有害"],
  ["我今天过得很开心", "无害"],
  ["没人喜欢你", "有害"],
  ["这令人无法接受。我想和经理谈谈。", "无害"],
];

const [inputs, outputs] = labeledTexts.reduce<
  [Array<{ input: string }>, Array<{ outputs: string }>]
>(
  ([inputs, outputs], item) => [
    [...inputs, { input: item[0] }],
    [...outputs, { outputs: item[1] }],
  ],
  [[], []]
);

const datasetName = "有害查询";
const toxicDataset = await langsmith.createDataset(datasetName);
await langsmith.createExamples({ inputs, outputs, datasetId: toxicDataset.id });

def correct(inputs: dict, outputs: dict, reference_outputs: dict) -> bool:
    return outputs["class"] == reference_outputs["label"]

import type { EvaluationResult } from "langsmith/evaluation";

function correct({
  outputs,
  referenceOutputs,
}: {
  outputs: Record<string, any>;
  referenceOutputs?: Record<string, any>;
}): EvaluationResult {
  const score = outputs.output === referenceOutputs?.outputs;
  return { key: "correct", score };
}

# 可选元数据，用于填充 UI 中的模型/提示词/工具列
EXPERIMENT_METADATA = {
    "models": [
        "openai:gpt-5.4-mini",
        {
            "id": ["langchain", "chat_models", "openai", "ChatOpenAI"],
            "lc": 1,
            "type": "constructor",
            "kwargs": {"model_name": "gpt-5.5", "temperature": 0.2},
        },
    ],
    "prompts": ["my-org/my-eval-prompt:abc12345"],
    "tools": [
        {
            "name": "web_search",
            "description": "在网络上搜索信息",
            "parameters": {
                "type": "object",
                "properties": {"query": {"type": "string"}},
                "required": ["query"],
            },
        },
    ],
}

# 也可以直接使用 'evaluate' 函数：
# from langsmith import evaluate; evaluate(...)
results = ls_client.evaluate(
    toxicity_classifier,
    data=dataset.name,
    evaluators=[correct],
    experiment_prefix="gpt-5.4-mini, baseline",  # 可选，实验名称前缀
    description="测试基准系统。",  # 可选，实验描述
    max_concurrency=4,  # 可选，添加并发
    metadata=EXPERIMENT_METADATA,  # 可选，用于填充 UI 中的模型/提示词/工具列
)

import { evaluate } from "langsmith/evaluation";

// 可选元数据，用于填充 UI 中的模型/提示词/工具列
const EXPERIMENT_METADATA = {
  models: [
    "openai:gpt-5.4-mini",
    {
      id: ["langchain", "chat_models", "openai", "ChatOpenAI"],
      lc: 1,
      type: "constructor",
      kwargs: { model_name: "gpt-5.5", temperature: 0.2 },
    },
  ],
  prompts: ["my-org/my-eval-prompt:abc12345"],
  tools: [
    {
      name: "web_search",
      description: "在网络上搜索信息",
      parameters: {
        type: "object",
        properties: { query: { type: "string" } },
        required: ["query"],
      },
    },
  ],
};

await evaluate((inputs) => toxicityClassifier(inputs["input"]), {
  data: datasetName,
  evaluators: [correct],
  experimentPrefix: "gpt-5.4-mini, baseline",  // 可选，实验名称前缀
  maxConcurrency: 4, // 可选，添加并发
  metadata: EXPERIMENT_METADATA,  // 可选，用于填充 UI 中的模型/提示词/工具列
});