接入自制OCR新工具时出现的问题 #356
Description
我在使用贵平台时出现了pydantic的问题
具体报错如下：
Traceback (most recent call last): File "/home/aistudio/deploy/llm_agent/api/common_tools/ocr_tools.py", line 26, in <module> class CustomOCRInput(ToolParameterView): File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_model_construction.py", line 200, in __new__ set_model_fields(cls, bases, config_wrapper, types_namespace) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_model_construction.py", line 469, in set_model_fields fields, class_vars = collect_model_fields(cls, bases, config_wrapper, types_namespace, typevars_map=typevars_map) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_fields.py", line 132, in collect_model_fields type_hints = get_cls_type_hints_lenient(cls, types_namespace) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_typing_extra.py", line 228, in get_cls_type_hints_lenient hints[name] = eval_type_lenient(value, globalns, localns) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_typing_extra.py", line 240, in eval_type_lenient return eval_type_backport(value, globalns, localns) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_typing_extra.py", line 264, in eval_type_backport return typing._eval_type( # type: ignore File "/home/aistudio/external-libraries/agent/lib/python3.10/typing.py", line 327, in _eval_type return t._evaluate(globalns, localns, recursive_guard) File "/home/aistudio/external-libraries/agent/lib/python3.10/typing.py", line 693, in _evaluate type_ = _type_check( File "/home/aistudio/external-libraries/agent/lib/python3.10/typing.py", line 176, in _type_check raise TypeError(f"{msg} Got {arg!r:.100}.") TypeError: Forward references must evaluate to types. Got <module 'PIL.Image' from '/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/PIL/I. (/home/aistudio/external-libraries/agent) aistudio@jupyter-2163331-8134354:~/deploy/llm_agent/api/common_tools$ python ocr_tools.py Traceback (most recent call last): File "/home/aistudio/deploy/llm_agent/api/common_tools/ocr_tools.py", line 7, in <module> from erniebot_agent.file import LocalFile ImportError: cannot import name 'LocalFile' from 'erniebot_agent.file' (/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/file/__init__.py) (/home/aistudio/external-libraries/agent) aistudio@jupyter-2163331-8134354:~/deploy/llm_agent/api/common_tools$ python ocr_tools.py Traceback (most recent call last): File "/home/aistudio/deploy/llm_agent/api/common_tools/ocr_tools.py", line 26, in <module> class CustomOCRInput(ToolParameterView): File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_model_construction.py", line 205, in __new__ complete_model_class( File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_model_construction.py", line 534, in complete_model_class schema = cls.__get_pydantic_core_schema__(cls, handler) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/main.py", line 642, in __get_pydantic_core_schema__ return handler(source) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_schema_generation_shared.py", line 83, in __call__ schema = self._handler(source_type) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 512, in generate_schema schema = self._generate_schema_inner(obj) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 784, in _generate_schema_inner return self._model_schema(obj) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 591, in _model_schema {k: self._generate_md_field_schema(k, v, decorators) for k, v in fields.items()}, File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 591, in <dictcomp> {k: self._generate_md_field_schema(k, v, decorators) for k, v in fields.items()}, File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 947, in _generate_md_field_schema common_field = self._common_field_schema(name, field_info, decorators) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 1134, in _common_field_schema schema = self._apply_annotations( File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 1890, in _apply_annotations schema = get_inner_schema(source_type) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_schema_generation_shared.py", line 83, in __call__ schema = self._handler(source_type) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 1871, in inner_handler schema = self._generate_schema_inner(obj) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 789, in _generate_schema_inner return self.match_type(obj) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 875, in match_type return self._unknown_type_schema(obj) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/pydantic/_internal/_generate_schema.py", line 415, in _unknown_type_schema raise PydanticSchemaGenerationError( **pydantic.errors.PydanticSchemaGenerationError: Unable to generate pydantic-core schema for <class 'erniebot_agent.file.local_file.LocalFile'>. Set arbitrary_types_allowed=Truein the model_config to ignore this error or implementget_pydantic_core_schema` on your type to fully support it.
If you got this error by calling handler() within __get_pydantic_core_schema__ then you likely need to call handler.generate_schema(<some type>) since we do not call __get_pydantic_core_schema__ on <some type> otherwise to avoid infinite recursion.
For further information visit https://errors.pydantic.dev/2.8/u/schema-for-unknown-type**
`
相关代码如下（具体的token与部分request逻辑我已经隐藏，应该不会影响到问题的判断）
`from future import annotations
import requests
from typing import Any, Dict, Type, List
from pydantic import Field
from erniebot_agent.tools.base import Tool
from erniebot_agent.file.local_file import LocalFile
from erniebot_agent.tools.schema import ToolParameterView
from erniebot_agent.agents.function_agent import FunctionAgent
from erniebot_agent.chat_models import ERNIEBot
from erniebot_agent.memory import WholeMemory
from erniebot_agent.file import GlobalFileManagerHandler
def image_path_to_base64(img):
#转换图像为base64
...
class CustomOCRInput(ToolParameterView):
img: LocalFile = Field(description="待识别的图像")
class CustomOCROutput(ToolParameterView):
result: str = Field(description="识别的文字结果")
class CustomOCRTool(Tool):
description: str = "识别输入图像中的文字"
input_type: Type[ToolParameterView] = CustomOCRInput
output_type: Type[ToolParameterView] = CustomOCROutput
def __init__(self) -> None:
super().__init__()
async def __call__(self, img: File) -> Dict[str, Any]:
url = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
data = image_path_to_base64(img)
# 发送 POST 请求
data = {
'img': data,
}
response = requests.post(url, data=data)
# 处理返回的图像数据
result = ""
if response.status_code == 200:
reply = response.json()
for block in reply:
result += block["transcription"]
result += "\n"
return {"result": result}
import asyncio
import os
async def a():
os.environ["EB_AGENT_ACCESS_TOKEN"] = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
file_manager = GlobalFileManagerHandler().get()
agent = FunctionAgent(ERNIEBot("ernie-3.5"), tools=[CustomOCRTool()], memory=WholeMemory())
data = await file_manager.create_file_from_path(file_path="/home/aistudio/deploy/llm_agent/api/img_12.jpg", file_type="local")
result = await agent.run("告诉我图片上讲了什么？", files=[data])
print(result)
asyncio.run(a())
`
相关环境如下：
平台为百度飞浆AI Studio提供的BML CodeLab配置（CPU）。
下面是我个人的一点看法：
从报错上来说，似乎是pydantic试图对LocalFile类做一点处理（我不是很了解这个），然后该类有一些方法没实现导致出现了问题？
同时 GlobalFileManagerHandler().get()与FunctionAgent(ERNIEBot("ernie-3.5"), tools=[CustomOCRTool()], memory=WholeMemory())
去掉了原教程中的await参数，因为也会报错（判断为GlobalFileManagerHandler().get()用不到异步，去掉后没有报错）
我尝试过声明arbitrary_types_allowed=True在相关的工具Input子类中
但是没有效果，会出现新的错误：
File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/agents/agent.py", line 195, in run_llm raise e File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/agents/agent.py", line 192, in run_llm llm_resp = await self._run_llm(messages, **(llm_opts or {})) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/agents/agent.py", line 324, in _run_llm functions = self._tool_manager.get_tool_schemas() File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/tools/tool_manager.py", line 73, in get_tool_schemas return [tool.function_call_schema() for tool in self._tools.values()] File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/tools/tool_manager.py", line 73, in <listcomp> return [tool.function_call_schema() for tool in self._tools.values()] File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/tools/base.py", line 80, in function_call_schema inputs["parameters"] = self.input_type.function_call_schema() File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/tools/schema.py", line 339, in function_call_schema return cls.to_openapi_dict() File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/tools/schema.py", line 321, in to_openapi_dict properties[field_name] = dict(get_field_openapi_property(field_info)) File "/home/aistudio/external-libraries/agent/lib/python3.10/site-packages/erniebot_agent/tools/schema.py", line 181, in get_field_openapi_property elif field_info.annotation is not None and issubclass(field_info.annotation, Enum): TypeError: issubclass() arg 1 must be a class
上面的堆栈报错中个人认为比较重要的部分
代码如下：
class CustomOCRInput(ToolParameterView): class Config: arbitrary_types_allowed = True local_file: 'erniebot_agent.file.local_file.LocalFile' img: LocalFile = Field(description="待识别的图像")
我已经黔驴技穷了，希望能够有大佬能够高抬贵手帮个忙，感谢感谢！！！