Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions guides/openai.md
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,44 @@ response.usage.cost

Responses API server-side tools may also appear in `response.message.tool_calls` as builtin records (for example `web_search_call` or `file_search_call`). They are preserved for observability, but the provider already executed them: do not replay them as local tool calls. `ReqLLM.Response.classify/1` and `ReqLLM.StreamResponse.classify/1` treat builtin-only responses as final answers.

### Code Interpreter (Responses API)

Models using the Responses API support the Code Interpreter tool, which runs Python code in a sandboxed container. Pass the tool as a map and ReqLLM will forward it unchanged to OpenAI:

```elixir
{:ok, response} = ReqLLM.generate_text(
"openai:gpt-5-mini",
"What is the factorial of 12804/53 + 300? Solve with Python.",
tools: [%{
"type" => "code_interpreter",
"container" => %{"type" => "auto", "memory_limit" => "4g"}
}]
)

# Access the raw code interpreter output items
response.provider_meta["code_interpreter"]["items"]
#=> [
#=> %{
#=> "type" => "code_interpreter_call",
#=> "code" => "from fractions import Fraction...",
#=> "status" => "completed",
#=> ...
#=> }
#=> ]

# Access code interpreter usage
response.usage.tool_usage.code_interpreter
#=> %{count: 1, unit: :call}
```

The container value may also be an existing container ID string:

```elixir
tools: [%{"type" => "code_interpreter", "container" => "cntr_abc123"}]
```

Code Interpreter is a server-side builtin: the provider executes the code and returns the result items. Do not replay them as local tool calls. `ReqLLM.Response.classify/1` treats these responses as final answers.

### Image Generation

Image generation costs are tracked separately:
Expand Down
104 changes: 80 additions & 24 deletions lib/req_llm/providers/openai/responses_api.ex
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,17 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do
- Token limits use `max_output_tokens` instead of `max_tokens`
- Tool choice format: `{type: "function", name: "tool_name"}`
- Reasoning effort: `{effort: "high"}` format
- **Code Interpreter**: tool maps such as `%{"type" => "code_interpreter", "container" => ...}`
are passed through unchanged to OpenAI. Both object containers
(`%{"type" => "auto", ...}`) and string container IDs are supported.
This is Responses-API-only; Chat Completions does not support the
`code_interpreter` tool type.

## Code Interpreter

Raw `code_interpreter_*` output items from the response are collected in
`response.provider_meta["code_interpreter"]["items"]` and are excluded from
normal text and function tool-call extraction.

## Decoding

Expand Down Expand Up @@ -59,13 +70,14 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do
require Logger
require ReqLLM.Debug, as: Debug

@builtin_tool_types ~w(web_search web_search_preview file_search mcp x_search)
@builtin_tool_types ~w(web_search web_search_preview file_search mcp x_search code_interpreter)
@tool_usage_type_atoms %{
"web_search" => :web_search,
"web_search_preview" => :web_search_preview,
"file_search" => :file_search,
"mcp" => :mcp,
"x_search" => :x_search
"x_search" => :x_search,
"code_interpreter" => :code_interpreter
}
@tool_call_atom_keys %{
"web_search_call" => :web_search_call,
Expand Down Expand Up @@ -266,6 +278,8 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do
meta =
maybe_put_reasoning_details(meta, extract_reasoning_details_from_segments(response_output))

meta = merge_code_interpreter_meta(meta, response_output)

meta = merge_response_provider_meta(meta, data["response"] || %{})

[ReqLLM.StreamChunk.meta(meta)]
Expand Down Expand Up @@ -306,6 +320,20 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do

defp merge_response_provider_meta(meta, _), do: meta

defp merge_code_interpreter_meta(meta, response_output) when is_list(response_output) do
items = extract_code_interpreter_items(response_output)

if items == [] do
meta
else
provider_meta = Map.get(meta, :provider_meta, %{})
updated = put_code_interpreter_meta(provider_meta, items)
Map.put(meta, :provider_meta, updated)
end
end

defp merge_code_interpreter_meta(meta, _), do: meta

defp drop_blanks(map) do
map
|> Enum.reject(fn {_key, value} -> value in [nil, ""] end)
Expand Down Expand Up @@ -1142,19 +1170,24 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do
end

type when is_binary(type) ->
if Map.has_key?(@tool_call_atom_keys, type) do
[
ReqLLM.StreamChunk.meta(%{
builtin_tool_started: %{
id: item["id"] || item[:id] || item["call_id"] || item[:call_id],
name: type,
index: stream_output_index(data),
started_at_unix_nano: System.system_time(:nanosecond)
}
})
]
else
[]
cond do
code_interpreter_item?(item) ->
[ReqLLM.StreamChunk.meta(%{code_interpreter_item: item})]

Map.has_key?(@tool_call_atom_keys, type) ->
[
ReqLLM.StreamChunk.meta(%{
builtin_tool_started: %{
id: item["id"] || item[:id] || item["call_id"] || item[:call_id],
name: type,
index: stream_output_index(data),
started_at_unix_nano: System.system_time(:nanosecond)
}
})
]

true ->
[]
end

_ ->
Expand All @@ -1175,19 +1208,22 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do
defp handle_output_item_done(_, _), do: []

defp handle_output_item_done_item(item, data, state) do
case item["type"] || item[:type] do
"function_call" ->
type = item["type"] || item[:type]

cond do
type == "function_call" ->
handle_function_call_item_done(item, data, state)

"message" ->
type == "message" ->
handle_message_item_done(item, data, state)

type when is_binary(type) ->
if Map.has_key?(@tool_call_atom_keys, type),
do: handle_builtin_call_item_done(item, data, state, type),
else: []
code_interpreter_item?(item) ->
[ReqLLM.StreamChunk.meta(%{code_interpreter_item: item})]

_ ->
is_binary(type) and Map.has_key?(@tool_call_atom_keys, type) ->
handle_builtin_call_item_done(item, data, state, type)

true ->
[]
end
end
Expand Down Expand Up @@ -1645,6 +1681,7 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do
thinking = aggregate_reasoning_segments(output_segments)
tool_calls = extract_tool_calls_from_segments(output_segments)
reasoning_details = extract_reasoning_details_from_segments(output_segments)
code_interpreter_items = extract_code_interpreter_items(output_segments)

base_usage = %{
input_tokens: get_in(body, ["usage", "input_tokens"]) || 0,
Expand Down Expand Up @@ -1682,7 +1719,10 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do
|> Map.drop(["id", "model", "output_text", "output", "usage"])
|> Map.put("api_type", "responses")

provider_meta = Map.merge(base_provider_meta, object_meta)
provider_meta =
base_provider_meta
|> Map.merge(object_meta)
|> put_code_interpreter_meta(code_interpreter_items)

response = %ReqLLM.Response{
id: body["id"] || "unknown",
Expand Down Expand Up @@ -2026,6 +2066,22 @@ defmodule ReqLLM.Providers.OpenAI.ResponsesAPI do

defp extract_summary_text(_), do: nil

defp extract_code_interpreter_items(segments) when is_list(segments) do
Enum.filter(segments, &code_interpreter_item?/1)
end

defp extract_code_interpreter_items(_), do: []

defp code_interpreter_item?(%{"type" => "code_interpreter" <> _}), do: true
defp code_interpreter_item?(%{type: "code_interpreter" <> _}), do: true
defp code_interpreter_item?(_), do: false

defp put_code_interpreter_meta(provider_meta, []), do: provider_meta

defp put_code_interpreter_meta(provider_meta, items) when is_list(items) do
Map.put(provider_meta, "code_interpreter", %{"items" => items})
end

defp normalize_arguments_json(nil), do: "{}"
defp normalize_arguments_json(""), do: "{}"

Expand Down
Loading