feat(tools): image reference processing and native provider support (#1251)

* feat(tools): implement image reference processing and native provider support

- Support OpenAI image edits via both Multipart form-data and JSON payloads
- Automatically append reference image descriptions to prompt under [Reference Image Roles]
- Support downloading image URLs for Gemini native image generation
- Deduplicate reference images to optimize API request size
- Add unit tests for Codex, DashScope, MiniMax, BytePlus and local/remote path resolution

* fix(tools): SSRF-guard reference-image URL downloads in create_image

downloadImageBytes fetched caller-supplied ref_images[].url with a plain
http.Client and unbounded io.ReadAll — no SSRF validation, redirect
policy, or size cap, letting the gateway dial loopback/private/metadata
hosts or read arbitrarily large responses.

- Validate the URL via security.Validate and pin the resolved IP, then
  download through security.NewSafeClient (pinned dial, no redirects).
- Cap the response with a bounded read (refImageMaxBytes, 20 MB).
- Reject non-HTTP(S) reference URLs up front (file://, data:, gopher://)
  so provider-forwarded URLs stay HTTP(S)-only; document the trust
  boundary between gateway-side fetch and provider-forwarded URLs.
- Add regression tests: blocked loopback/private/metadata, unfollowed
  redirect, oversized response, and non-http(s) scheme rejection.

Author: thotam

internal/providers/codex_native_image.go

@@ -57,28 +57,51 @@ func (p *CodexProvider) GenerateImage(ctx context.Context, req NativeImageReques
 // so stream is always true. Final assembly happens in parseNativeImageSSE which scans the
 // event stream for response.output_item.done (image item) or response.completed output walk.
 func (p *CodexProvider) buildNativeImageRequestBody(model string, req NativeImageRequest) map[string]any {
+	tool := map[string]any{
+		"type":          "image_generation",
+		"action":        "generate",
+		"model":         req.ImageModel,
+		"output_format": req.OutputFormat,
+		"size":          SizeFromAspect(req.AspectRatio),
+	}
+	
+	contentParts := []map[string]any{}
+
+	for _, img := range req.RefImages {
+		if img.Base64 != "" {
+			refMime := img.MimeType
+			if refMime == "" {
+				refMime = "image/png"
+			}
+			contentParts = append(contentParts, map[string]any{
+				"type":      "input_image",
+				"image_url": fmt.Sprintf("data:%s;base64,%s", refMime, img.Base64),
+			})
+		} else if img.URL != "" {
+			contentParts = append(contentParts, map[string]any{
+				"type":      "input_image",
+				"image_url": img.URL,
+			})
+		}
+	}
+
+	contentParts = append(contentParts, map[string]any{
+		"type": "input_text",
+		"text": req.Prompt,
+	})
+
 	return map[string]any{
 		"model":        model,
 		"stream":       true,
 		"store":        false,
 		"instructions": "Generate an image matching the user's description using the image_generation tool. Return only the image; do not describe it in text.",
 		"input": []any{
 			map[string]any{
-				"role": "user",
-				"content": []map[string]any{
-					{"type": "input_text", "text": req.Prompt},
-				},
-			},
-		},
-		"tools": []map[string]any{
-			{
-				"type":          "image_generation",
-				"action":        "generate",
-				"model":         req.ImageModel,
-				"output_format": req.OutputFormat,
-				"size":          SizeFromAspect(req.AspectRatio),
+				"role":    "user",
+				"content": contentParts,
 			},
 		},
+		"tools": []map[string]any{tool},
 		"tool_choice": map[string]any{
 			"type": "image_generation",
 		},

internal/providers/codex_native_image_test.go

@@ -344,3 +344,140 @@ func TestSizeFromAspect(t *testing.T) {
 		}
 	}
 }
+
+// TestCodexGenerateImage_WithReferenceImage verifies that passing a single RefImage
+// embeds the image in the input content and does not populate input_reference in tools[0].
+func TestCodexGenerateImage_WithReferenceImage(t *testing.T) {
+	var captured []byte
+	server := mockImageServer(t, &captured)
+	defer server.Close()
+
+	p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
+	p.retryConfig.Attempts = 1
+
+	req := NativeImageRequest{
+		Model:        "gpt-image-2",
+		Prompt:       "A red circle",
+		RefImages: []RefImage{
+			{
+				URL: "https://example.com/ref.png",
+			},
+		},
+		AspectRatio:  "1:1",
+		OutputFormat: "png",
+	}
+
+	_, err := p.GenerateImage(context.Background(), req)
+	if err != nil {
+		t.Fatalf("GenerateImage returned error: %v", err)
+	}
+
+	var body map[string]any
+	if err := json.Unmarshal(captured, &body); err != nil {
+		t.Fatalf("unmarshal captured body: %v", err)
+	}
+
+	// 1. Verify input contains input_image and input_text in user content
+	inputs, ok := body["input"].([]any)
+	if !ok || len(inputs) != 1 {
+		t.Fatalf("body[input]: expected []any length 1, got %T len %d", body["input"], len(inputs))
+	}
+	userMsg, ok := inputs[0].(map[string]any)
+	if !ok {
+		t.Fatalf("inputs[0] is not a map")
+	}
+	contents, ok := userMsg["content"].([]any)
+	if !ok || len(contents) != 2 {
+		t.Fatalf("content: expected []any length 2, got %T len %d", userMsg["content"], len(contents))
+	}
+
+	imgPart, ok := contents[0].(map[string]any)
+	if !ok {
+		t.Fatalf("contents[0] is not a map")
+	}
+	if imgPart["type"] != "input_image" || imgPart["image_url"] != "https://example.com/ref.png" {
+		t.Errorf("expected input_image with url, got: %v", imgPart)
+	}
+
+	textPart, ok := contents[1].(map[string]any)
+	if !ok {
+		t.Fatalf("contents[1] is not a map")
+	}
+	if textPart["type"] != "input_text" || textPart["text"] != "A red circle" {
+		t.Errorf("expected input_text with prompt, got: %v", textPart)
+	}
+
+	// 2. Verify tools[0] does not contain input_reference
+	tools, ok := body["tools"].([]any)
+	if !ok || len(tools) != 1 {
+		t.Fatalf("tools shape invalid")
+	}
+	tool, ok := tools[0].(map[string]any)
+	if !ok {
+		t.Fatalf("tools[0] not map")
+	}
+	if _, has := tool["input_reference"]; has {
+		t.Error("image_generation tool must not contain 'input_reference' field")
+	}
+}
+
+// TestCodexGenerateImage_WithMultipleReferenceImages verifies that passing multiple RefImages
+// embeds all of them in the input content in the correct order.
+func TestCodexGenerateImage_WithMultipleReferenceImages(t *testing.T) {
+	var captured []byte
+	server := mockImageServer(t, &captured)
+	defer server.Close()
+
+	p := NewCodexProvider("codex-test", &staticTokenSource{token: "tok"}, server.URL, "gpt-image-2")
+	p.retryConfig.Attempts = 1
+
+	req := NativeImageRequest{
+		Model:        "gpt-image-2",
+		Prompt:       "A red circle",
+		RefImages: []RefImage{
+			{
+				URL: "https://example.com/ref1.png",
+			},
+			{
+				Base64:   "b64data",
+				MimeType: "image/jpeg",
+			},
+		},
+		AspectRatio:  "1:1",
+		OutputFormat: "png",
+	}
+
+	_, err := p.GenerateImage(context.Background(), req)
+	if err != nil {
+		t.Fatalf("GenerateImage returned error: %v", err)
+	}
+
+	var body map[string]any
+	if err := json.Unmarshal(captured, &body); err != nil {
+		t.Fatalf("unmarshal captured body: %v", err)
+	}
+
+	inputs, _ := body["input"].([]any)
+	userMsg, _ := inputs[0].(map[string]any)
+	contents, _ := userMsg["content"].([]any)
+
+	// Expected: 2 images + 1 text prompt = 3 content parts
+	if len(contents) != 3 {
+		t.Fatalf("content: expected []any length 3, got len %d", len(contents))
+	}
+
+	imgPart1, _ := contents[0].(map[string]any)
+	if imgPart1["type"] != "input_image" || imgPart1["image_url"] != "https://example.com/ref1.png" {
+		t.Errorf("expected first input_image with url, got: %v", imgPart1)
+	}
+
+	imgPart2, _ := contents[1].(map[string]any)
+	if imgPart2["type"] != "input_image" || imgPart2["image_url"] != "data:image/jpeg;base64,b64data" {
+		t.Errorf("expected second input_image with base64 data url, got: %v", imgPart2)
+	}
+
+	textPart, _ := contents[2].(map[string]any)
+	if textPart["type"] != "input_text" || textPart["text"] != "A red circle" {
+		t.Errorf("expected input_text with prompt, got: %v", textPart)
+	}
+}

internal/providers/native_image.go

@@ -61,6 +61,18 @@ type NativeImageRequest struct {
 
 	// OutputFormat is the desired image format: "png" (default), "jpg", "webp".
 	OutputFormat string
+
+	// RefImages contains the list of reference images.
+	RefImages []RefImage
+}
+
+// RefImage represents a single reference image for image-to-image or styling tasks.
+type RefImage struct {
+	Data     []byte
+	Base64   string
+	MimeType string
+	URL      string
+	Strength float64
 }
 
 // NativeImageResult holds the result of a native image generation call.