如何运行成对评估

from langchain_classic import hub
from langchain.chat_models import init_chat_model
from langsmith import evaluate

# 查看提示：https://smith.langchain.com/hub/langchain-ai/pairwise-evaluation-2
prompt = hub.pull("langchain-ai/pairwise-evaluation-2")
model = init_chat_model("gpt-5.5")
chain = prompt | model

def ranked_preference(inputs: dict, outputs: list[dict]) -> list:
    # 假设示例输入包含 'question' 键，实验
    # 输出包含 'answer' 键。
    response = chain.invoke({
        "question": inputs["question"],
        "answer_a": outputs[0].get("answer", "N/A"),
        "answer_b": outputs[1].get("answer", "N/A"),
    })
    if response["Preference"] == 1:
        scores = [1, 0]
    elif response["Preference"] == 2:
        scores = [0, 1]
    else:
        scores = [0, 0]
    return scores

evaluate(
    ("experiment-1", "experiment-2"),  # 替换为你的实验名称/ID
    evaluators=[ranked_preference],
    randomize_order=True,
    max_concurrency=4,
)

import { evaluate} from "langsmith/evaluation";
import { Run } from "langsmith/schemas";
import { wrapOpenAI } from "langsmith/wrappers";
import OpenAI from "openai";
import { z } from "zod";

const openai = wrapOpenAI(new OpenAI());

async function rankedPreference({
  inputs,
  runs,
}: {
  inputs: Record<string, any>;
  runs: Run[];
}) {
  const scores: Record<string, number> = {};
  const [runA, runB] = runs;
  if (!runA || !runB) throw new Error("至少需要两个运行");

  const payload = {
    question: inputs.question,
    answer_a: runA?.outputs?.output ?? "N/A",
    answer_b: runB?.outputs?.output ?? "N/A",
  };

  const output = await openai.chat.completions.create({
    model: "gpt-4-turbo",
    messages: [
      {
        role: "system",
        content: [
          "请充当公正的评判者，评估下面两个 AI 助手针对用户问题提供的响应质量。",
          "你应选择更好地遵循用户指令并回答用户问题的助手。",
          "你的评估应考虑响应的帮助性、相关性、准确性、深度、创造性和详细程度等因素。",
          "请先比较两个响应，并给出简短的解释。",
          "避免任何位置偏差，并确保响应的呈现顺序不会影响你的决定。",
          "不要让响应的长度影响你的评估。不要偏爱某些助手名称。请尽可能保持客观。",
        ].join(" "),
      },
      {
        role: "user",
        content: [
          `[用户问题] ${payload.question}`,
          `[助手 A 回答的开始] ${payload.answer_a} [助手 A 回答的结束]`,
          `[助手 B 回答的开始] ${payload.answer_b} [助手 B 回答的结束]`,
        ].join("\n\n"),
      },
    ],
    tool_choice: {
      type: "function",
      function: { name: "Score" },
    },
    tools: [
      {
        type: "function",
        function: {
          name: "Score",
          description: [
            `请在给出解释后，严格遵循以下格式输出最终判定：`,
            `如果根据上述因素助手 A 的回答更好，则输出 "1"。`,
            `如果根据上述因素助手 B 的回答更好，则输出 "2"。`,
            `如果两者平局，则输出 "0"。`,
          ].join(" "),
          parameters: {
            type: "object",
            properties: {
              Preference: {
                type: "integer",
                description: "偏好哪个助手的回答？",
              },
            },
          },
        },
      },
    ],
  });

  const { Preference } = z
    .object({ Preference: z.number() })
    .parse(
      JSON.parse(output.choices[0].message.tool_calls[0].function.arguments)
    );

  if (Preference === 1) {
    scores[runA.id] = 1;
    scores[runB.id] = 0;
  } else if (Preference === 2) {
    scores[runA.id] = 0;
    scores[runB.id] = 1;
  } else {
    scores[runA.id] = 0;
    scores[runB.id] = 0;
  }

  return { key: "ranked_preference", scores };
}

await evaluate(["earnest-name-40", "reflecting-pump-91"], {
  evaluators: [rankedPreference],
});