Init Project

pkg/vllmkit/client.go

@@ -0,0 +1,845 @@
+// Package vllmkit talks to the production vLLM OpenAI-compatible
+// inference servers that host Qwen 3.5 (chat + tool calling) and a
+// separate embedding model. Shared by the agent service (ChatStream)
+// and the knowledge service (Embed) — the same internal/llm.Client the
+// monolith had, split out since both of those bounded contexts need it
+// and neither should depend on the other.
+//
+// This package previously spoke Ollama's native /api/chat and
+// /api/embed protocol directly against a single local daemon. It now
+// speaks the OpenAI /v1/chat/completions and /v1/embeddings wire
+// format that `vllm serve` exposes, against two independently scaled,
+// independently deployed vLLM servers — one per model, since a single
+// vLLM engine process serves exactly one model. See
+// ENKLY_VLLM_CHAT_URL / ENKLY_VLLM_EMBED_URL in cmd/server/main.go and
+// k8s/11-vllm.yaml for how those servers are configured and deployed.
+package vllmkit
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	// DefaultChatModel is the served model name the chat vLLM
+	// deployment must be started with (`vllm serve ... --served-model-name
+	// qwen3.5`). vLLM matches the request body's "model" field against
+	// that flag's value, not against a Hugging Face repo id, so this
+	// is a deployment-chosen alias, not a model identifier — keep it
+	// in sync with k8s/11-vllm.yaml's --served-model-name.
+	DefaultChatModel = "qwen3.5"
+	// DefaultEmbedModel is the served model name for the embedding
+	// vLLM deployment. vLLM's pooling/embedding model support covers
+	// nomic-ai/nomic-embed-text-v1.5 (what this deployment actually
+	// runs; it needs `--trust-remote-code` since Nomic's config uses
+	// custom modeling code) — small, fast, and a good CPU/GPU-both
+	// fit. Update the `vector(768)` dimension in schema.sql first if
+	// you point this at a model with a different output size.
+	DefaultEmbedModel = "nomic-embed-text"
+)
+
+// Options mirrors the subset of the OpenAI-compatible chat-completions
+// request body that's worth tuning per call. Unlike Ollama's
+// num_ctx/num_gpu/num_thread, context length and GPU/tensor-parallel
+// layout are not per-request knobs under vLLM — they're fixed at
+// server start (--max-model-len, --tensor-parallel-size,
+// --gpu-memory-utilization; see k8s/11-vllm.yaml) and apply to every
+// request the deployment serves. Zero values are omitted from the wire
+// request, so the server's own defaults apply unless explicitly
+// overridden — see the ENKLY_CHAT_* env vars wired up in
+// cmd/server/main.go.
+type Options struct {
+	Temperature float64 // sampling temperature (temperature)
+	TopP        float64 // nucleus sampling cutoff (top_p)
+	MaxTokens   int     // max tokens to generate this turn (max_tokens); 0 = server default
+}
+
+// Client is a thin wrapper around two vLLM OpenAI-compatible HTTP
+// APIs: one serving the chat model, one serving the embedding model.
+type Client struct {
+	// ChatBaseURL / EmbedBaseURL point at each vLLM deployment's
+	// OpenAI-compatible router, e.g. "https://vllm-chat.enkly.org/v1" and
+	// "https://vllm-embed.enkly.org/v1". They are deliberately separate
+	// fields (rather than one shared host) because a chat model and
+	// an embedding model are different vLLM engine processes in
+	// production — usually on different Services/Deployments with
+	// different GPU shapes — not two endpoints behind the same daemon
+	// the way Ollama modeled it.
+	ChatBaseURL  string
+	EmbedBaseURL string
+	ChatModel    string
+	EmbedModel   string
+	HTTP         *http.Client
+
+	// APIKey, when set, is sent as "Authorization: Bearer <APIKey>" on
+	// every request — vLLM's --api-key flag. Leave empty for a
+	// cluster-internal deployment with no auth in front of it (the
+	// Service is only reachable from inside the mesh/namespace); set
+	// it for anything exposed beyond that boundary. Set via
+	// ENKLY_VLLM_API_KEY.
+	APIKey string
+
+	// Think controls extended "thinking" on chat requests via vLLM's
+	// chat_template_kwargs extra-body extension: "" leaves it to the
+	// chat template's own default (Qwen3's thinking-capable template
+	// defaults to thinking on), "true"/"false" (and the usual
+	// on/off/yes/no spellings) force it on/off. A "low"/"medium"/
+	// "high" effort string (see pkg/effort) is also accepted for
+	// compatibility with the Anthropic-style effort vocabulary the
+	// rest of the agent uses — vLLM/Qwen3's reasoning toggle itself is
+	// a single on/off switch with no server-side notion of graded
+	// effort, so any non-empty, non-"false" level here just turns
+	// thinking on; depth is instead shaped by Options.MaxTokens and
+	// agentrun's own round/compaction budgets (see effort.Profile).
+	// When on, the reasoning trace streams back separately from the
+	// answer, in OpenAI's "reasoning_content" delta field
+	// (ChatEvent.Thinking) — requires the chat deployment to be
+	// started with a reasoning parser, e.g. `--reasoning-parser
+	// qwen3`. Set via ENKLY_CHAT_THINK.
+	Think string
+	// ChatOptions is sent as the top-level temperature/top_p/max_tokens
+	// fields of the chat-completions request.
+	ChatOptions Options
+
+	// embedCache memoizes Embed() results within this process's
+	// lifetime, keyed by (model, text) hash. Embedding the exact same
+	// string twice is common — repeated search_codebase queries within
+	// a conversation, or an unchanged chunk re-indexed after a
+	// no-op edit that rag.Store's content-hash check didn't already
+	// short-circuit — and skipping the round trip to vLLM for a
+	// repeat is a pure latency win with no correctness downside, since
+	// embeddings are a deterministic function of (model, text).
+	embedCache *embedCache
+}
+
+// New builds a Client pointed at the given chat and embed vLLM
+// deployments, with connection reuse tuned for the request pattern
+// here: a handful of long-lived streaming chat requests plus frequent,
+// short embedding calls, each against its own vLLM host. Either URL
+// may be left empty if this Client will only ever be used for the
+// other call (e.g. the knowledge service never calls ChatStream, so it
+// passes "" for chatBaseURL).
+func New(chatBaseURL, embedBaseURL, chatModel, embedModel string) *Client {
+	if chatBaseURL == "" {
+		chatBaseURL = "http://localhost:8000/v1"
+	}
+	if embedBaseURL == "" {
+		embedBaseURL = "http://localhost:8001/v1"
+	}
+	if chatModel == "" {
+		chatModel = DefaultChatModel
+	}
+	if embedModel == "" {
+		embedModel = DefaultEmbedModel
+	}
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   10 * time.Second,
+			KeepAlive: 60 * time.Second,
+		}).DialContext,
+		// Two vLLM hosts, but chat + embed + concurrent requests (see
+		// agent.go's parallel tool execution, which can itself fan out
+		// concurrent search_codebase embed calls) can be in flight
+		// together — a small pool avoids a fresh TCP+HTTP handshake on
+		// every call, a real chunk of latency in practice even against
+		// a fast in-cluster backend.
+		MaxIdleConns:        64,
+		MaxIdleConnsPerHost: 32,
+		IdleConnTimeout:     90 * time.Second,
+	}
+	return &Client{
+		ChatBaseURL:  chatBaseURL,
+		EmbedBaseURL: embedBaseURL,
+		ChatModel:    chatModel,
+		EmbedModel:   embedModel,
+		HTTP: &http.Client{
+			Transport: transport,
+			// No overall timeout: chat streams can legitimately run for
+			// minutes while the agent loop executes several tool calls.
+			// Callers should thread a context deadline through ctx instead.
+		},
+		embedCache: newEmbedCache(500),
+	}
+}
+
+func (c *Client) authHeader(req *http.Request) {
+	if c.APIKey != "" {
+		req.Header.Set("Authorization", "Bearer "+c.APIKey)
+	}
+}
+
+// ---------------------------------------------------------------------
+// Chat (streaming, with tool calling) — OpenAI /v1/chat/completions
+// ---------------------------------------------------------------------
+
+// wireToolCall is the OpenAI wire shape for a tool call, both on a
+// complete assistant message (echoed back into history) and,
+// incrementally, on a streamed delta (see toolCallAccumulator).
+type wireToolCall struct {
+	Index    *int                 `json:"index,omitempty"` // present only on streamed deltas
+	ID       string               `json:"id,omitempty"`
+	Type     string               `json:"type,omitempty"` // always "function"
+	Function wireToolCallFunction `json:"function"`
+}
+
+type wireToolCallFunction struct {
+	Name string `json:"name,omitempty"`
+	// Arguments is a JSON-encoded string on the wire (OpenAI's
+	// function-calling contract), not a nested object the way Ollama's
+	// native API sent it — toWireMessages/fromWireToolCalls marshal
+	// and unmarshal it at this package's boundary so the rest of the
+	// codebase still deals in map[string]any.
+	Arguments string `json:"arguments,omitempty"`
+}
+
+type wireMessage struct {
+	Role       string         `json:"role"`
+	Content    *string        `json:"content"` // nil (omitted) for an assistant message that is tool_calls-only
+	ToolCalls  []wireToolCall `json:"tool_calls,omitempty"`
+	ToolCallID string         `json:"tool_call_id,omitempty"` // role="tool" only
+	Name       string         `json:"name,omitempty"`         // role="tool" only; some parsers want the tool name too
+}
+
+type wireToolDef struct {
+	Type     string           `json:"type"` // always "function"
+	Function wireToolFunction `json:"function"`
+}
+
+type wireToolFunction struct {
+	Name        string         `json:"name"`
+	Description string         `json:"description"`
+	Parameters  map[string]any `json:"parameters"`
+}
+
+type chatRequest struct {
+	Model    string        `json:"model"`
+	Messages []wireMessage `json:"messages"`
+	Tools    []wireToolDef `json:"tools,omitempty"`
+	Stream   bool          `json:"stream"`
+	// ChatTemplateKwargs is vLLM's extra-body extension point for
+	// per-request chat-template flags; "enable_thinking" is how
+	// Qwen3's template (and vLLM's qwen3 reasoning parser) take the
+	// thinking on/off toggle. Opaque to the OpenAI spec itself, so
+	// this is marshaled as a bare map rather than a typed field.
+	ChatTemplateKwargs map[string]any `json:"chat_template_kwargs,omitempty"`
+
+	Temperature float64 `json:"temperature,omitempty"`
+	TopP        float64 `json:"top_p,omitempty"`
+	MaxTokens   int     `json:"max_tokens,omitempty"`
+}
+
+// chatStreamChunk is one Server-Sent-Events "data:" line of an OpenAI
+// streaming chat-completions response.
+type chatStreamChunk struct {
+	Choices []struct {
+		Delta struct {
+			Content          string         `json:"content"`
+			ReasoningContent string         `json:"reasoning_content"`
+			ToolCalls        []wireToolCall `json:"tool_calls"`
+		} `json:"delta"`
+		FinishReason *string `json:"finish_reason"`
+	} `json:"choices"`
+	// Error is set when a vLLM server streams a mid-response failure
+	// as a normal 200 SSE line rather than an HTTP error status (seen
+	// e.g. when the tool-call parser can't make sense of the model's
+	// raw output partway through generation).
+	Error *wireError `json:"error"`
+}
+
+// ErrUpstreamUnavailable wraps a transport-level failure to reach a
+// vLLM deployment at all — connection refused, DNS failure, dial/TLS
+// timeout — as opposed to vLLM responding with an error status or a
+// malformed body. This is exactly the signature of a vllm-chat/
+// vllm-embed pod that's Pending, still cold-starting (model load, see
+// k8s/11-vllm.yaml's startupProbe), or mid-rollout (Recreate strategy
+// briefly has zero replicas) — not a request vLLM itself rejected.
+//
+// Callers use IsUpstreamUnavailable to tell the two apart: the agent
+// service turns this into a short "model is starting up" message for
+// the end user instead of a raw dial error (see
+// services/agent/internal/application/service.go), and the knowledge
+// service's gRPC layer maps it onto codes.Unavailable (safe to retry)
+// instead of codes.Internal (opaque failure).
+var ErrUpstreamUnavailable = errors.New("vllm: upstream unreachable")
+
+// IsUpstreamUnavailable reports whether err is, or wraps,
+// ErrUpstreamUnavailable.
+func IsUpstreamUnavailable(err error) bool { return errors.Is(err, ErrUpstreamUnavailable) }
+
+// looksTransient reports whether err is a transport-level failure
+// (connection refused, DNS, dial/handshake timeout, connection reset)
+// worth a couple of quick retries, as opposed to an error vLLM itself
+// returned after accepting the connection.
+func looksTransient(err error) bool {
+	if err == nil {
+		return false
+	}
+	var netErr net.Error
+	if errors.As(err, &netErr) {
+		return true // covers dial timeout/refused, DNS failure, TLS handshake failure, etc.
+	}
+	msg := strings.ToLower(err.Error())
+	return strings.Contains(msg, "connection refused") ||
+		strings.Contains(msg, "no such host") ||
+		strings.Contains(msg, "connection reset")
+}
+
+// retryDelays bounds how long doWithRetry rides out a transient
+// failure before giving up: short and few on purpose. This exists to
+// smooth over a brief reconnect window — e.g. a Recreate rollout's old
+// pod having just terminated — not to mask a genuinely Pending pod,
+// which is a scheduling problem no amount of HTTP retrying fixes (see
+// k8s/scripts/gpu-preflight.sh for that diagnosis).
+var retryDelays = []time.Duration{200 * time.Millisecond, 600 * time.Millisecond}
+
+// doWithRetry POSTs buf as JSON to url, retrying only on a transient
+// transport failure (see looksTransient) — never after a response has
+// actually been received, since a non-2xx status or a stream already
+// in progress isn't safe or meaningful to blindly resend.
+func (c *Client) doWithRetry(ctx context.Context, url string, buf []byte, sse bool) (*http.Response, error) {
+	for attempt := 0; ; attempt++ {
+		req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf))
+		if err != nil {
+			return nil, err
+		}
+		req.Header.Set("Content-Type", "application/json")
+		if sse {
+			req.Header.Set("Accept", "text/event-stream")
+		}
+		c.authHeader(req)
+
+		resp, err := c.HTTP.Do(req)
+		if err == nil {
+			return resp, nil
+		}
+		if attempt >= len(retryDelays) || !looksTransient(err) {
+			return nil, err
+		}
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(retryDelays[attempt]):
+		}
+	}
+}
+
+type wireError struct {
+	Message string `json:"message"`
+	Type    string `json:"type"`
+	Code    string `json:"code"`
+}
+
+func (e *wireError) Error() string {
+	if e == nil {
+		return ""
+	}
+	if e.Type != "" {
+		return fmt.Sprintf("%s: %s", e.Type, e.Message)
+	}
+	return e.Message
+}
+
+// errMalformedToolArgs is wrapped into the error ChatStream reports
+// when vLLM's tool-call parser hands back an "arguments" string that
+// doesn't parse as JSON. This is this package's own client-side
+// detection of a broken tool-call round (vLLM never surfaces this as a
+// distinct HTTP/wire error itself) — see IsToolCallParseError.
+var errMalformedToolArgs = errors.New("vllm: tool call arguments were not valid JSON")
+
+// toolCallParseErrPattern matches the family of messages vLLM's
+// OpenAI-compatible server reports when a configured tool-call parser
+// (hermes, llama3_json, qwen3, mistral, …) fails to extract a
+// structured call from the model's raw generation — e.g. "Failed to
+// parse tool calls", "could not parse function call", "invalid tool
+// call arguments produced by model". Server-side wording differs by
+// parser plugin and vLLM version, so this matches on the stable,
+// shared vocabulary ("tool call" / "function call" alongside "parse"
+// or "invalid") rather than one exact phrasing.
+func toolCallParseErrPattern(msg string) bool {
+	msg = strings.ToLower(msg)
+	mentionsCall := strings.Contains(msg, "tool call") || strings.Contains(msg, "tool_call") ||
+		strings.Contains(msg, "function call") || strings.Contains(msg, "function_call")
+	mentionsFailure := strings.Contains(msg, "parse") || strings.Contains(msg, "invalid") || strings.Contains(msg, "malformed")
+	return mentionsCall && mentionsFailure
+}
+
+// IsToolCallParseError reports whether err is a transient failure to
+// extract a structured tool call from the model's output — either
+// vLLM's own tool-call parser rejecting the raw generation server-side
+// (see toolCallParseErrPattern), or this client failing to decode the
+// "arguments" JSON string vLLM did return (errMalformedToolArgs). Both
+// are almost always a one-off decoding hiccup — something in the
+// model's raw output didn't match what the configured parser expected
+// — rather than a reason to give up the whole agent turn. Callers
+// (agentrun.Agent.Run) use this to retry the round, and if it keeps
+// recurring, to simplify the most recent tool results before one final
+// retry.
+//
+// This replaces IsTemplateParseError, which detected the equivalent
+// failure mode under Ollama's native chat-template/XML tool-call
+// encoding — a protocol vLLM's OpenAI-compatible server doesn't use,
+// so that detection logic (XML syntax errors, angle-bracket tag
+// sniffing) no longer applies, but the role it plays in the retry loop
+// is unchanged.
+func IsToolCallParseError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, errMalformedToolArgs) {
+		return true
+	}
+	return toolCallParseErrPattern(err.Error())
+}
+
+// thinkWire maps a Think setting onto vLLM's chat_template_kwargs
+// extra body. It returns nil when thinking should be left to the
+// template's own default (omitting chat_template_kwargs entirely), and
+// otherwise a map with "enable_thinking" set to true/false.
+//
+// override, when non-empty, takes precedence over c.Think for this one
+// call — see ChatStream's think parameter. This is what lets a caller
+// (agentrun.Agent's ThinkPolicy) dial reasoning on or off on a
+// per-round basis — e.g. thinking on for the first, planning round of
+// an agentic tool-use turn, thinking off for the rounds that just
+// digest a tool result.
+func thinkWire(setting, override string) map[string]any {
+	s := override
+	if s == "" {
+		s = setting
+	}
+	switch strings.ToLower(strings.TrimSpace(s)) {
+	case "":
+		return nil
+	case "false", "off", "0", "no":
+		return map[string]any{"enable_thinking": false}
+	default:
+		// "true"/"on"/"1"/"yes" and the effort levels ("low", "medium",
+		// "high", …) all just mean "thinking on" — see Think's doc
+		// comment for why vLLM/Qwen3 has no graded-effort equivalent.
+		return map[string]any{"enable_thinking": true}
+	}
+}
+
+// ChatEvent is one item streamed back from ChatStream: either a token of
+// assistant content, a batch of tool calls the model wants executed, or
+// the terminal done/error signal.
+type ChatEvent struct {
+	Token     string
+	Thinking  string // a chunk of the model's reasoning trace (when thinking is on)
+	ToolCalls []ToolCallRequest
+	Done      bool
+	Err       error
+}
+
+// ChatStream sends the full message history (plus tool definitions) to
+// the chat vLLM deployment and streams the reply back over the
+// returned channel. The channel is closed after a Done or Err event.
+//
+// think overrides c.Think for this one call when non-empty (see
+// thinkWire); pass "" to just use the Client's configured default.
+func (c *Client) ChatStream(ctx context.Context, history []Message, tools []ToolDef, think string) (<-chan ChatEvent, error) {
+	reqBody := chatRequest{
+		Model:              c.ChatModel,
+		Messages:           toWireMessages(history),
+		Tools:              toWireTools(tools),
+		Stream:             true,
+		ChatTemplateKwargs: thinkWire(c.Think, think),
+	}
+	reqBody.Temperature = c.ChatOptions.Temperature
+	reqBody.TopP = c.ChatOptions.TopP
+	reqBody.MaxTokens = c.ChatOptions.MaxTokens
+
+	buf, err := json.Marshal(reqBody)
+	if err != nil {
+		return nil, fmt.Errorf("llm: marshal chat request: %w", err)
+	}
+
+	resp, err := c.doWithRetry(ctx, c.ChatBaseURL+"/chat/completions", buf, true)
+	if err != nil {
+		if looksTransient(err) {
+			return nil, fmt.Errorf("llm: chat request failed (is vLLM running at %s?): %w: %w", c.ChatBaseURL, err, ErrUpstreamUnavailable)
+		}
+		return nil, fmt.Errorf("llm: chat request failed (is vLLM running at %s?): %w", c.ChatBaseURL, err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		defer resp.Body.Close()
+		body := readErrorBody(resp)
+		if resp.StatusCode == http.StatusServiceUnavailable {
+			return nil, fmt.Errorf("llm: vllm returned %s: %s: %w", resp.Status, body, ErrUpstreamUnavailable)
+		}
+		return nil, fmt.Errorf("llm: vllm returned %s: %s", resp.Status, body)
+	}
+
+	events := make(chan ChatEvent, 16)
+
+	go func() {
+		defer resp.Body.Close()
+		defer close(events)
+
+		// send delivers an event unless the consumer has gone away (ctx
+		// cancelled), so this goroutine can never block forever on a
+		// channel nobody is reading.
+		send := func(ev ChatEvent) bool {
+			select {
+			case events <- ev:
+				return true
+			case <-ctx.Done():
+				return false
+			}
+		}
+
+		acc := newToolCallAccumulator()
+
+		scanner := bufio.NewScanner(resp.Body)
+		scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
+
+		sawDone := false
+		for scanner.Scan() {
+			line := bytes.TrimSpace(scanner.Bytes())
+			if len(line) == 0 {
+				continue
+			}
+			data, ok := bytes.CutPrefix(line, []byte("data:"))
+			if !ok {
+				// Non-"data:" SSE lines (comments, "event:", blank
+				// keep-alives) carry nothing this client needs.
+				continue
+			}
+			data = bytes.TrimSpace(data)
+			if string(data) == "[DONE]" {
+				sawDone = true
+				break
+			}
+
+			var chunk chatStreamChunk
+			if err := json.Unmarshal(data, &chunk); err != nil {
+				send(ChatEvent{Err: fmt.Errorf("llm: decode stream chunk: %w", err)})
+				return
+			}
+			if chunk.Error != nil {
+				send(ChatEvent{Err: fmt.Errorf("llm: vllm error: %w", chunk.Error)})
+				return
+			}
+			if len(chunk.Choices) == 0 {
+				continue
+			}
+			choice := chunk.Choices[0]
+
+			if choice.Delta.ReasoningContent != "" {
+				if !send(ChatEvent{Thinking: choice.Delta.ReasoningContent}) {
+					return
+				}
+			}
+			if choice.Delta.Content != "" {
+				if !send(ChatEvent{Token: choice.Delta.Content}) {
+					return
+				}
+			}
+			if len(choice.Delta.ToolCalls) > 0 {
+				acc.add(choice.Delta.ToolCalls)
+			}
+
+			if choice.FinishReason != nil {
+				if acc.len() > 0 {
+					calls, err := acc.finish()
+					if err != nil {
+						send(ChatEvent{Err: fmt.Errorf("llm: %w: %w", errMalformedToolArgs, err)})
+						return
+					}
+					if !send(ChatEvent{ToolCalls: calls}) {
+						return
+					}
+				}
+				sawDone = true
+				send(ChatEvent{Done: true})
+				return
+			}
+		}
+
+		if err := scanner.Err(); err != nil {
+			send(ChatEvent{Err: fmt.Errorf("llm: reading stream: %w", err)})
+			return
+		}
+		// A [DONE] sentinel with no finish_reason chunk (or EOF with
+		// neither) means vLLM dropped the stream before completing the
+		// reply (crash, OOM-kill, network cut) — surface it rather
+		// than treating a truncated reply as a complete one.
+		if !sawDone && ctx.Err() == nil {
+			send(ChatEvent{Err: fmt.Errorf("llm: vllm closed the stream before finishing the reply")})
+			return
+		}
+		if sawDone {
+			send(ChatEvent{Done: true})
+		}
+	}()
+
+	return events, nil
+}
+
+func readErrorBody(resp *http.Response) string {
+	body := make([]byte, 2048)
+	n, _ := resp.Body.Read(body)
+	raw := body[:n]
+
+	var envelope struct {
+		Error wireError `json:"error"`
+	}
+	if err := json.Unmarshal(raw, &envelope); err == nil && envelope.Error.Message != "" {
+		return envelope.Error.Error()
+	}
+	return string(raw)
+}
+
+// ---------------------------------------------------------------------
+// toolCallAccumulator: reassembles a streamed, index-keyed sequence of
+// partial tool_calls deltas into complete calls. The OpenAI streaming
+// contract sends a tool call's id/name once (on the chunk that first
+// introduces that index) and its "arguments" JSON string in arbitrary-
+// length fragments across however many subsequent chunks it takes to
+// finish generating them — chunk boundaries do not align with JSON
+// token boundaries at all, so nothing is parseable until finish().
+// ---------------------------------------------------------------------
+
+type toolCallAccumulator struct {
+	order []int
+	byIdx map[int]*accumulatingCall
+}
+
+type accumulatingCall struct {
+	id, name string
+	args     strings.Builder
+}
+
+func newToolCallAccumulator() *toolCallAccumulator {
+	return &toolCallAccumulator{byIdx: map[int]*accumulatingCall{}}
+}
+
+func (a *toolCallAccumulator) len() int { return len(a.byIdx) }
+
+func (a *toolCallAccumulator) add(deltas []wireToolCall) {
+	for _, d := range deltas {
+		idx := 0
+		if d.Index != nil {
+			idx = *d.Index
+		}
+		call, ok := a.byIdx[idx]
+		if !ok {
+			call = &accumulatingCall{}
+			a.byIdx[idx] = call
+			a.order = append(a.order, idx)
+		}
+		if d.ID != "" {
+			call.id = d.ID
+		}
+		if d.Function.Name != "" {
+			call.name = d.Function.Name
+		}
+		call.args.WriteString(d.Function.Arguments)
+	}
+}
+
+// finish parses every accumulated call's argument string as JSON and
+// returns them in the order their indices first appeared. An empty
+// arguments string (a tool with no parameters) is treated as "{}",
+// matching what every vLLM tool-call parser actually emits for a
+// no-arg call.
+func (a *toolCallAccumulator) finish() ([]ToolCallRequest, error) {
+	sort.Ints(a.order)
+	out := make([]ToolCallRequest, 0, len(a.order))
+	for _, idx := range a.order {
+		call := a.byIdx[idx]
+		raw := strings.TrimSpace(call.args.String())
+		if raw == "" {
+			raw = "{}"
+		}
+		var args map[string]any
+		if err := json.Unmarshal([]byte(raw), &args); err != nil {
+			return nil, fmt.Errorf("tool %q (id %s): %w", call.name, call.id, err)
+		}
+		out = append(out, ToolCallRequest{ID: call.id, Name: call.name, Arguments: args})
+	}
+	return out, nil
+}
+
+func toWireMessages(history []Message) []wireMessage {
+	out := make([]wireMessage, 0, len(history))
+	for _, m := range history {
+		content := m.Content
+		wm := wireMessage{
+			Role:       string(m.Role),
+			Content:    &content,
+			ToolCallID: m.ToolCallID,
+			Name:       m.ToolName,
+		}
+		// An assistant message that only carries tool calls (no reply
+		// text) sends content as null, not "" — some vLLM tool-call
+		// parsers treat an empty-string content as "the model also
+		// said something" and choke on it.
+		if m.Role == RoleAssistant && len(m.ToolCalls) > 0 && m.Content == "" {
+			wm.Content = nil
+		}
+		for _, tc := range m.ToolCalls {
+			args := tc.Arguments
+			if args == nil {
+				args = map[string]any{}
+			}
+			argsJSON, err := json.Marshal(args)
+			if err != nil {
+				// Arguments is always built from values this process
+				// itself constructed (see toolcatalog/executor), so a
+				// marshal failure here would be a programming error,
+				// not a runtime condition — fall back to "{}" rather
+				// than drop the whole request.
+				argsJSON = []byte("{}")
+			}
+			wm.ToolCalls = append(wm.ToolCalls, wireToolCall{
+				ID:       tc.ID,
+				Type:     "function",
+				Function: wireToolCallFunction{Name: tc.Name, Arguments: string(argsJSON)},
+			})
+		}
+		out = append(out, wm)
+	}
+	return out
+}
+
+func toWireTools(tools []ToolDef) []wireToolDef {
+	out := make([]wireToolDef, 0, len(tools))
+	for _, t := range tools {
+		out = append(out, wireToolDef{
+			Type: "function",
+			Function: wireToolFunction{
+				Name:        t.Name,
+				Description: t.Description,
+				Parameters:  t.Parameters,
+			},
+		})
+	}
+	return out
+}
+
+// ---------------------------------------------------------------------
+// Embeddings — OpenAI /v1/embeddings
+// ---------------------------------------------------------------------
+
+type embedRequest struct {
+	Model string `json:"model"`
+	Input string `json:"input"`
+}
+
+type embedResponse struct {
+	Data []struct {
+		Embedding []float32 `json:"embedding"`
+		Index     int       `json:"index"`
+	} `json:"data"`
+}
+
+// Embed returns the embedding vector for a single string, using the
+// embed vLLM deployment's /v1/embeddings endpoint. Identical (model,
+// text) pairs are served from an in-memory cache instead of
+// round-tripping to vLLM again — see the embedCache doc comment on
+// Client.
+func (c *Client) Embed(ctx context.Context, text string) ([]float32, error) {
+	if c.embedCache != nil {
+		if v, ok := c.embedCache.get(c.EmbedModel, text); ok {
+			return v, nil
+		}
+	}
+
+	reqBody := embedRequest{Model: c.EmbedModel, Input: text}
+	buf, err := json.Marshal(reqBody)
+	if err != nil {
+		return nil, fmt.Errorf("llm: marshal embed request: %w", err)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+
+	resp, err := c.doWithRetry(ctx, c.EmbedBaseURL+"/embeddings", buf, false)
+	if err != nil {
+		if looksTransient(err) {
+			return nil, fmt.Errorf("llm: embed request failed (is vLLM running at %s?): %w: %w", c.EmbedBaseURL, err, ErrUpstreamUnavailable)
+		}
+		return nil, fmt.Errorf("llm: embed request failed (is vLLM running at %s?): %w", c.EmbedBaseURL, err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body := readErrorBody(resp)
+		if resp.StatusCode == http.StatusServiceUnavailable {
+			return nil, fmt.Errorf("llm: vllm embed returned %s: %s: %w", resp.Status, body, ErrUpstreamUnavailable)
+		}
+		return nil, fmt.Errorf("llm: vllm embed returned %s: %s", resp.Status, body)
+	}
+
+	var parsed embedResponse
+	if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
+		return nil, fmt.Errorf("llm: decode embed response: %w", err)
+	}
+	if len(parsed.Data) == 0 {
+		return nil, fmt.Errorf("llm: embed response contained no vectors")
+	}
+
+	vec := parsed.Data[0].Embedding
+	if c.embedCache != nil {
+		c.embedCache.put(c.EmbedModel, text, vec)
+	}
+	return vec, nil
+}
+
+// ---------------------------------------------------------------------
+// embedCache: small bounded in-memory cache for Embed() results
+// ---------------------------------------------------------------------
+
+type embedCache struct {
+	mu     sync.Mutex
+	cap    int
+	order  []string // insertion order, for cheap FIFO eviction
+	values map[string][]float32
+}
+
+func newEmbedCache(capacity int) *embedCache {
+	return &embedCache{cap: capacity, values: make(map[string][]float32, capacity)}
+}
+
+func embedCacheKey(model, text string) string {
+	h := sha256.Sum256([]byte(model + "\x00" + text))
+	return hex.EncodeToString(h[:])
+}
+
+func (c *embedCache) get(model, text string) ([]float32, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	v, ok := c.values[embedCacheKey(model, text)]
+	return v, ok
+}
+
+func (c *embedCache) put(model, text string, vec []float32) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	key := embedCacheKey(model, text)
+	if _, exists := c.values[key]; !exists {
+		if len(c.order) >= c.cap {
+			// Evict the oldest entry (simple FIFO — good enough for a
+			// latency optimization, not a correctness-critical cache).
+			oldest := c.order[0]
+			c.order = c.order[1:]
+			delete(c.values, oldest)
+		}
+		c.order = append(c.order, key)
+	}
+	c.values[key] = vec
+}

pkg/vllmkit/client_test.go

@@ -0,0 +1,210 @@
+package vllmkit
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestThinkWire(t *testing.T) {
+	cases := []struct {
+		name              string
+		setting, override string
+		wantEnable        *bool // nil means "chat_template_kwargs omitted entirely"
+	}{
+		{"both empty", "", "", nil},
+		{"setting only, bool true", "true", "", boolPtr(true)},
+		{"setting only, bool false", "false", "", boolPtr(false)},
+		{"setting only, level", "high", "", boolPtr(true)},
+		{"override wins over setting", "low", "high", boolPtr(true)},
+		{"override alone", "", "medium", boolPtr(true)},
+		{"override explicit false wins", "true", "false", boolPtr(false)},
+		{"case/space insensitive", "  TRUE ", "", boolPtr(true)},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := thinkWire(tc.setting, tc.override)
+			if tc.wantEnable == nil {
+				if got != nil {
+					t.Errorf("thinkWire(%q, %q) = %#v, want nil", tc.setting, tc.override, got)
+				}
+				return
+			}
+			gotEnable, ok := got["enable_thinking"].(bool)
+			if !ok || gotEnable != *tc.wantEnable {
+				t.Errorf("thinkWire(%q, %q) = %#v, want enable_thinking=%v", tc.setting, tc.override, got, *tc.wantEnable)
+			}
+		})
+	}
+}
+
+func boolPtr(b bool) *bool { return &b }
+
+// TestChatStream_ThinkOverride confirms the per-call think parameter
+// reaches the wire request's chat_template_kwargs.enable_thinking,
+// taking precedence over the Client's configured default exactly as
+// thinkWire specifies — this is the knob agentrun.Agent.ThinkPolicy
+// relies on to vary reasoning on or off by round.
+func TestChatStream_ThinkOverride(t *testing.T) {
+	var gotKwargs map[string]any
+	sawKwargs := false
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var req chatRequest
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			t.Fatalf("decode request: %v", err)
+		}
+		gotKwargs = req.ChatTemplateKwargs
+		sawKwargs = req.ChatTemplateKwargs != nil
+
+		w.Header().Set("Content-Type", "text/event-stream")
+		fmt.Fprintf(w, "data: %s\n\n", mustJSON(finishedChunk()))
+		fmt.Fprintf(w, "data: [DONE]\n\n")
+	}))
+	defer srv.Close()
+
+	run := func(clientThink, callThink string) (map[string]any, bool) {
+		c := New(srv.URL, "", "test-model", "")
+		c.Think = clientThink
+		events, err := c.ChatStream(context.Background(), []Message{{Role: RoleUser, Content: "hi"}}, nil, callThink)
+		if err != nil {
+			t.Fatalf("ChatStream: %v", err)
+		}
+		for ev := range events {
+			if ev.Err != nil {
+				t.Fatalf("stream error: %v", ev.Err)
+			}
+		}
+		return gotKwargs, sawKwargs
+	}
+
+	if got, saw := run("", ""); saw || got != nil {
+		t.Errorf("client default empty, no override: got kwargs=%#v saw=%v, want omitted", got, saw)
+	}
+	if got, _ := run("low", ""); got["enable_thinking"] != true {
+		t.Errorf("client default only: got kwargs=%#v, want enable_thinking=true", got)
+	}
+	if got, _ := run("false", "high"); got["enable_thinking"] != true {
+		t.Errorf("override should win: got kwargs=%#v, want enable_thinking=true", got)
+	}
+	if got, _ := run("true", "false"); got["enable_thinking"] != false {
+		t.Errorf("override alone (explicit false): got kwargs=%#v, want enable_thinking=false", got)
+	}
+}
+
+func finishedChunk() chatStreamChunk {
+	var c chatStreamChunk
+	c.Choices = make([]struct {
+		Delta struct {
+			Content          string         `json:"content"`
+			ReasoningContent string         `json:"reasoning_content"`
+			ToolCalls        []wireToolCall `json:"tool_calls"`
+		} `json:"delta"`
+		FinishReason *string `json:"finish_reason"`
+	}, 1)
+	reason := "stop"
+	c.Choices[0].FinishReason = &reason
+	return c
+}
+
+func mustJSON(v any) string {
+	b, err := json.Marshal(v)
+	if err != nil {
+		panic(err)
+	}
+	return string(b)
+}
+
+// TestChatStream_StreamsTokens is a sanity check that the SSE-based
+// stream reader in ChatStream still behaves with an
+// bufio.Scanner-friendly response (guards against accidentally
+// breaking the "data: " framing while touching this file).
+func TestChatStream_StreamsTokens(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream")
+		bw := bufio.NewWriter(w)
+
+		write := func(content string, done bool) {
+			var c chatStreamChunk
+			c.Choices = make([]struct {
+				Delta struct {
+					Content          string         `json:"content"`
+					ReasoningContent string         `json:"reasoning_content"`
+					ToolCalls        []wireToolCall `json:"tool_calls"`
+				} `json:"delta"`
+				FinishReason *string `json:"finish_reason"`
+			}, 1)
+			c.Choices[0].Delta.Content = content
+			if done {
+				reason := "stop"
+				c.Choices[0].FinishReason = &reason
+			}
+			fmt.Fprintf(bw, "data: %s\n\n", mustJSON(c))
+		}
+		write("hel", false)
+		write("lo", true)
+		fmt.Fprint(bw, "data: [DONE]\n\n")
+		bw.Flush()
+	}))
+	defer srv.Close()
+
+	c := New(srv.URL, "", "test-model", "")
+	events, err := c.ChatStream(context.Background(), []Message{{Role: RoleUser, Content: "hi"}}, nil, "")
+	if err != nil {
+		t.Fatalf("ChatStream: %v", err)
+	}
+	var got string
+	for ev := range events {
+		if ev.Err != nil {
+			t.Fatalf("stream error: %v", ev.Err)
+		}
+		got += ev.Token
+	}
+	if got != "hello" {
+		t.Errorf("got tokens %q, want %q", got, "hello")
+	}
+}
+
+// TestToolCallAccumulator_Streamed confirms incremental, index-keyed
+// tool_calls deltas (arguments split across several chunks, as a real
+// vLLM server streams them) reassemble into one complete call.
+func TestToolCallAccumulator_Streamed(t *testing.T) {
+	idx := 0
+	acc := newToolCallAccumulator()
+	acc.add([]wireToolCall{{Index: &idx, ID: "call_1", Function: wireToolCallFunction{Name: "grep"}}})
+	acc.add([]wireToolCall{{Index: &idx, Function: wireToolCallFunction{Arguments: `{"pat`}}})
+	acc.add([]wireToolCall{{Index: &idx, Function: wireToolCallFunction{Arguments: `tern":"foo"}`}}})
+
+	calls, err := acc.finish()
+	if err != nil {
+		t.Fatalf("finish: %v", err)
+	}
+	if len(calls) != 1 {
+		t.Fatalf("got %d calls, want 1", len(calls))
+	}
+	if calls[0].ID != "call_1" || calls[0].Name != "grep" {
+		t.Errorf("got %+v, want id=call_1 name=grep", calls[0])
+	}
+	if calls[0].Arguments["pattern"] != "foo" {
+		t.Errorf("got arguments %#v, want pattern=foo", calls[0].Arguments)
+	}
+}
+
+func TestIsToolCallParseError(t *testing.T) {
+	if !IsToolCallParseError(fmt.Errorf("wrap: %w", errMalformedToolArgs)) {
+		t.Error("wrapped errMalformedToolArgs should be detected")
+	}
+	if !IsToolCallParseError(fmt.Errorf("llm: vllm error: BadRequestError: Failed to parse tool calls from model output")) {
+		t.Error("server-side tool-call parse failure should be detected")
+	}
+	if IsToolCallParseError(fmt.Errorf("llm: vllm returned 500 Internal Server Error: out of memory")) {
+		t.Error("unrelated server error should not be classified as a tool-call parse error")
+	}
+	if IsToolCallParseError(nil) {
+		t.Error("nil error should never be a tool-call parse error")
+	}
+}

pkg/vllmkit/types.go

@@ -0,0 +1,73 @@
+package vllmkit
+
+// Role, Message, ToolCallRequest and ToolDef are local equivalents of
+// the monolith's internal/models types — kept here, scoped to exactly
+// what the OpenAI-compatible chat-completions wire protocol vLLM
+// serves needs, so this package has no dependency beyond the Go
+// standard library. The agent service (the only caller of ChatStream)
+// translates its own domain messages to and from these at the
+// boundary; see services/agent/internal/infrastructure/agentrun.
+type Role string
+
+const (
+	RoleUser      Role = "user"
+	RoleAssistant Role = "assistant"
+	RoleSystem    Role = "system"
+	RoleTool      Role = "tool"
+)
+
+// Message is one turn in a conversation, as sent to/received from
+// vLLM's OpenAI-compatible /v1/chat/completions endpoint.
+type Message struct {
+	Role     Role
+	Content  string
+	ToolName string
+	Thinking string // reasoning trace (OpenAI wire: "reasoning_content"); sent back to the model, never persisted
+
+	// ToolCalls is set on an assistant message that requested one or
+	// more tool invocations — see ToolCallRequest.
+	ToolCalls []ToolCallRequest
+
+	// ToolCallID identifies, on a role="tool" message, which of the
+	// preceding assistant message's ToolCalls this is the result for.
+	// The OpenAI tool-calling contract requires every tool message to
+	// carry the matching call's id (unlike Ollama's native /api/chat,
+	// which correlated tool replies positionally with no id at all) —
+	// vLLM's tool-call parsers (hermes, llama3-json, qwen, …) reject a
+	// tool message with a missing/mismatched tool_call_id outright, so
+	// this must be populated whenever Role == RoleTool. Callers get it
+	// for free: ToolCallRequest.ID below is populated straight off the
+	// id vLLM assigned the call, and ToolResult carries it forward
+	// unchanged — see agentrun.Agent.Run, which copies
+	// ToolCallRequest.ID into the ToolResult it builds and then into
+	// this field when it appends the reply message.
+	ToolCallID string
+}
+
+// ToolCallRequest is a single tool invocation the model asked for. ID
+// is the id vLLM's response assigned the call (OpenAI wire:
+// tool_calls[].id) — callers must echo it back as ToolResult's
+// ToolCallID so the follow-up tool message can carry Message.ToolCallID.
+type ToolCallRequest struct {
+	ID        string
+	Name      string
+	Arguments map[string]any
+}
+
+// ToolResult is what gets fed back to the model after executing a
+// ToolCallRequest.
+type ToolResult struct {
+	ToolCallID string
+	Name       string
+	Content    string
+	IsError    bool
+}
+
+// ToolDef describes a single callable tool in the JSON-schema shape
+// the OpenAI-compatible function-calling API (and therefore vLLM)
+// expects.
+type ToolDef struct {
+	Name        string
+	Description string
+	Parameters  map[string]any // JSON Schema object
+}