使用多模态内容运行评估

import requests
import uuid
from pathlib import Path
from langsmith import Client

# 公开可用的测试文件
pdf_url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
wav_url = "https://openaiassets.blob.core.windows.net/$web/API/docs/audio/alloy.wav"
img_url = "https://www.w3.org/Graphics/PNG/nurbcup2si.png"

# 将文件获取为字节
pdf_bytes = requests.get(pdf_url).content
wav_bytes = requests.get(wav_url).content
img_bytes = requests.get(img_url).content

# 创建数据集
ls_client = Client()
dataset_name = "attachment-test-dataset"
dataset = ls_client.create_dataset(
  dataset_name=dataset_name,
  description="用于评估公开可用附件的测试数据集",
)

inputs = {
  "audio_question": "这段音频中有什么？",
  "image_question": "这张图像中有什么？",
}

outputs = {
  "audio_answer": "太阳从东方升起，在西方落下。这个简单的事实已经被人类观察了数千年。",
  "image_answer": "一个上面盖着毯子的马克杯。",
}

# 定义带附件的示例
example_id = uuid.uuid4()
example = {
  "id": example_id,
  "inputs": inputs,
  "outputs": outputs,
  "attachments": {
      "my_pdf": {"mime_type": "application/pdf", "data": pdf_bytes},
      "my_wav": {"mime_type": "audio/wav", "data": wav_bytes},
      "my_img": {"mime_type": "image/png", "data": img_bytes},
      # 通过本地文件路径指定附件的示例：
      # "my_local_img": {"mime_type": "image/png", "data": Path(__file__).parent / "my_local_img.png"},
  },
}

# 创建示例
ls_client.create_examples(
  dataset_id=dataset.id,
  examples=[example],
  # 如果希望从本地文件上传附件，请取消此标志的注释：
  # dangerously_allow_filesystem=True
)

import { Client } from "langsmith";
import { v4 as uuid4 } from "uuid";

// 公开可用的测试文件
const pdfUrl = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf";
const wavUrl = "https://openaiassets.blob.core.windows.net/$web/API/docs/audio/alloy.wav";
const pngUrl = "https://www.w3.org/Graphics/PNG/nurbcup2si.png";

// 获取文件为 ArrayBuffer 的辅助函数
async function fetchArrayBuffer(url: string): Promise<ArrayBuffer> {
  const response = await fetch(url);
  if (!response.ok) {
    throw new Error(`获取 ${url} 失败：${response.statusText}`);
  }
  return response.arrayBuffer();
}

// 将文件获取为 ArrayBuffer
const pdfArrayBuffer = await fetchArrayBuffer(pdfUrl);
const wavArrayBuffer = await fetchArrayBuffer(wavUrl);
const pngArrayBuffer = await fetchArrayBuffer(pngUrl);

// 创建 LangSmith 客户端（请确保已在环境变量中设置 LANGSMITH_API_KEY）
const langsmithClient = new Client();

// 创建唯一的数据集名称
const datasetName = "attachment-test-dataset:" + uuid4().substring(0, 8);

// 创建数据集
const dataset = await langsmithClient.createDataset(datasetName, {
  description: "用于评估公开可用附件的测试数据集",
});

// 定义带附件的示例
const exampleId = uuid4();
const example = {
  id: exampleId,
  inputs: {
      audio_question: "这段音频中有什么？",
      image_question: "这张图像中有什么？",
  },
  outputs: {
      audio_answer: "太阳从东方升起，在西方落下。这个简单的事实已经被人类观察了数千年。",
      image_answer: "一个上面盖着毯子的马克杯。",
  },
  attachments: {
    my_pdf: {
      mimeType: "application/pdf",
      data: pdfArrayBuffer
    },
    my_wav: {
      mimeType: "audio/wav",
      data: wavArrayBuffer
    },
    my_img: {
      mimeType: "image/png",
      data: pngArrayBuffer
    },
  },
};

// 将带附件的示例上传到数据集
await langsmithClient.uploadExamplesMultipart(dataset.id, [example]);

{
    "presigned_url": str,
    "mime_type": str,
    "reader": BinaryIO
}

from langsmith.wrappers import wrap_openai
import base64
from openai import OpenAI

client = wrap_openai(OpenAI())

# 定义使用附件的目标函数
def file_qa(inputs, attachments):
    # 从 reader 读取音频字节，并将其编码为 base64
    audio_reader = attachments["my_wav"]["reader"]
    audio_b64 = base64.b64encode(audio_reader.read()).decode('utf-8')

    audio_completion = client.chat.completions.create(
        model="gpt-4o-audio-preview",
        messages=[
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": inputs["audio_question"]
                    },
                    {
                        "type": "input_audio",
                        "input_audio": {
                            "data": audio_b64,
                            "format": "wav"
                        }
                    }
                ]
            }
        ]
    )

    # 大多数模型除了支持 base64 编码的图像外，也支持直接接收图像 URL
    # 你可以将文件的预签名 URL 直接传递给模型
    image_url = attachments["my_img"]["presigned_url"]
    image_completion = client.chat.completions.create(
        model="gpt-5.4-mini",
        messages=[
          {
            "role": "user",
            "content": [
              {"type": "text", "text": inputs["image_question"]},
              {
                "type": "image_url",
                "image_url": {
                  "url": image_url,
                },
              },
            ],
          }
        ],
    )

    return {
        "audio_answer": audio_completion.choices[0].message.content,
        "image_answer": image_completion.choices[0].message.content,
    }

{
  presigned_url: string,
  mime_type: string,
}

import OpenAI from "openai";
import { wrapOpenAI } from "langsmith/wrappers";

const client: any = wrapOpenAI(new OpenAI());

async function fileQA(inputs: Record<string, any>, config?: Record<string, any>) {
  const presignedUrl = config?.attachments?.["my_wav"]?.presigned_url;
  if (!presignedUrl) {
    throw new Error("未提供音频的预签名 URL。");
  }

  const response = await fetch(presignedUrl);
  if (!response.ok) {
    throw new Error(`获取音频失败：${response.statusText}`);
  }

  const arrayBuffer = await response.arrayBuffer();
  const uint8Array = new Uint8Array(arrayBuffer);
  const audioB64 = Buffer.from(uint8Array).toString("base64");

  const audioCompletion = await client.chat.completions.create({
    model: "gpt-4o-audio-preview",
    messages: [
      {
        role: "user",
        content: [
          { type: "text", text: inputs["audio_question"] },
          {
            type: "input_audio",
            input_audio: {
              data: audioB64,
              format: "wav",
            },
          },
        ],
      },
    ],
  });

  const imageUrl = config?.attachments?.["my_img"]?.presigned_url
  const imageCompletion = await client.chat.completions.create({
    model: "gpt-5.4-mini",
    messages: [
      {
        role: "user",
        content: [
          { type: "text", text: inputs["image_question"] },
          {
            type: "image_url",
            image_url: {
              url: imageUrl,
            },
          },
        ],
      },
    ],
  });

  return {
    audio_answer: audioCompletion.choices[0].message.content,
    image_answer: imageCompletion.choices[0].message.content,
  };
}