331 lines
12 KiB
TypeScript
331 lines
12 KiB
TypeScript
import type {
|
|
LlmContentBlock,
|
|
LlmStreamHandle,
|
|
LlmStreamOptions,
|
|
} from "../assistant/llm-protocol";
|
|
import { openSseRequest, type SseListener } from "./sse";
|
|
|
|
/** True only for Venice's official API root, never for lookalike hosts. */
|
|
export function isVeniceApiBaseUrl(input: string): boolean {
|
|
return normalizeOpenAiCompatibleUrl(input) === "https://api.venice.ai/api/v1";
|
|
}
|
|
|
|
/** Streaming client for OpenAI-compatible Chat Completions APIs. */
|
|
export type OpenAiCompatibleStreamOptions = LlmStreamOptions & {
|
|
/** Explicit API root, for example https://api.venice.ai/api/v1 or https://host/api. */
|
|
baseUrl: string;
|
|
/** Venice-only web search; never sent to other compatible providers. */
|
|
webSearch?: boolean;
|
|
};
|
|
|
|
type PendingToolCall = {
|
|
id: string;
|
|
name: string;
|
|
inputJson: string;
|
|
};
|
|
|
|
/** Remove an optional endpoint suffix, but never guess a provider's /v1 or /api prefix. */
|
|
export function normalizeOpenAiCompatibleUrl(input: string): string {
|
|
let url = input.trim().replace(/\/+$/, "");
|
|
url = url.replace(/\/chat\/completions$/i, "").replace(/\/+$/, "");
|
|
return /^https?:\/\/[^/\s?#@\\]+(?:\/[^\s?#\\]*)?$/i.test(url) ? url : "";
|
|
}
|
|
|
|
export function streamOpenAiCompatibleChat(options: OpenAiCompatibleStreamOptions): LlmStreamHandle {
|
|
const apiKey = options.apiKey.trim();
|
|
const baseUrl = normalizeOpenAiCompatibleUrl(options.baseUrl);
|
|
let request: { cancel(): void } | null = null;
|
|
let settled = false;
|
|
let text = "";
|
|
let finishReason: string | null = null;
|
|
const toolCalls = new Map<number, PendingToolCall>();
|
|
const providerMetadata: Record<string, any> = {};
|
|
const reasoningDetails = new ReasoningDetailsAccumulator();
|
|
|
|
const fail = (message: string) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
request?.cancel();
|
|
options.onError(message);
|
|
};
|
|
|
|
const done = () => {
|
|
if (settled) return;
|
|
const hasToolCalls = toolCalls.size > 0;
|
|
// Never execute calls from a truncated stream or a length-limited response.
|
|
if (hasToolCalls && finishReason !== "tool_calls" && finishReason !== "stop") {
|
|
fail("OpenAI-compatible stream ended before its tool calls completed");
|
|
return;
|
|
}
|
|
if (!text && !hasToolCalls) {
|
|
fail("OpenAI-compatible provider ended the turn without a reply");
|
|
return;
|
|
}
|
|
|
|
const content: LlmContentBlock[] = [];
|
|
if (text) content.push({ type: "text", text });
|
|
const details = reasoningDetails.entries();
|
|
if (details.length) providerMetadata.reasoning_details = details;
|
|
if (Object.keys(providerMetadata).length) {
|
|
content.push({ type: "provider_item", provider: "openai-compatible", item: providerMetadata });
|
|
}
|
|
for (const index of Array.from(toolCalls.keys()).sort((a, b) => a - b)) {
|
|
const call = toolCalls.get(index)!;
|
|
if (!call.name) {
|
|
fail("OpenAI-compatible provider returned a tool call without a name");
|
|
return;
|
|
}
|
|
let input: any = {};
|
|
if (call.inputJson.trim()) {
|
|
try {
|
|
input = JSON.parse(call.inputJson);
|
|
} catch {
|
|
fail("OpenAI-compatible provider returned malformed tool arguments");
|
|
return;
|
|
}
|
|
}
|
|
content.push({
|
|
type: "tool_use",
|
|
id: call.id || `call_${index + 1}`,
|
|
name: call.name,
|
|
input,
|
|
});
|
|
}
|
|
|
|
settled = true;
|
|
request?.cancel();
|
|
options.onDone({
|
|
text,
|
|
content,
|
|
stopReason: finishReason === "content_filter" ? "refusal"
|
|
: hasToolCalls ? "tool_use"
|
|
: finishReason === "length" ? "max_tokens" : "end_turn",
|
|
});
|
|
};
|
|
|
|
if (!baseUrl || !options.model.trim()) {
|
|
setTimeout(() => fail(!baseUrl
|
|
? "Set a valid OpenAI-compatible API URL"
|
|
: "Set an OpenAI-compatible model ID"), 0);
|
|
return { cancel: () => { settled = true; } };
|
|
}
|
|
|
|
const body: any = {
|
|
model: options.model.trim(),
|
|
messages: buildChatMessages(options),
|
|
stream: true,
|
|
};
|
|
// Let the provider choose its default unless a one-shot feature supplies a cap.
|
|
if (options.maxTokens !== undefined) body.max_tokens = options.maxTokens;
|
|
if (options.tools?.length) {
|
|
body.tools = options.tools.map((tool) => ({
|
|
type: "function",
|
|
function: {
|
|
name: tool.name,
|
|
description: tool.description,
|
|
parameters: tool.input_schema,
|
|
},
|
|
}));
|
|
}
|
|
// Provider-specific parameters must never leak to another endpoint, even
|
|
// when a saved Venice preference is still enabled.
|
|
if (options.webSearch === true && isVeniceApiBaseUrl(baseUrl)) {
|
|
body.venice_parameters = { enable_web_search: "on" };
|
|
}
|
|
// Do not send provider-specific reasoning fields by default. Compatible
|
|
// servers vary widely; advanced reasoning controls can be added separately.
|
|
|
|
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
|
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
|
|
|
|
const listener: SseListener = {
|
|
onLine: (line: string) => {
|
|
if (settled || !line.startsWith("data:")) return;
|
|
const payload = line.slice(5).trim();
|
|
if (!payload) return;
|
|
if (payload === "[DONE]") {
|
|
if (finishReason || (!toolCalls.size && text)) done();
|
|
else fail("OpenAI-compatible stream ended before the response completed");
|
|
return;
|
|
}
|
|
let event: any;
|
|
try {
|
|
event = JSON.parse(payload);
|
|
} catch {
|
|
return;
|
|
}
|
|
if (event?.error) {
|
|
fail(`OpenAI-compatible: ${String(event.error.message ?? event.error)}`);
|
|
return;
|
|
}
|
|
const choice = event?.choices?.[0];
|
|
if (!choice) return; // Usage and provider-metadata chunks may have no choices.
|
|
const delta = choice.delta ?? {};
|
|
if (typeof delta.content === "string" && delta.content) {
|
|
text += delta.content;
|
|
options.onTextDelta?.(delta.content, text);
|
|
}
|
|
// Reasoning is not displayed. Preserve provider metadata for replay in
|
|
// multi-turn tool loops when the service supplies it.
|
|
if (Array.isArray(delta.reasoning_details)) reasoningDetails.add(delta.reasoning_details);
|
|
if (delta.thought_signature !== undefined) providerMetadata.thought_signature = delta.thought_signature;
|
|
if (Array.isArray(delta.tool_calls)) {
|
|
delta.tool_calls.forEach((call: any, position: number) => {
|
|
const index = Number.isInteger(call?.index) ? Number(call.index) : position;
|
|
const pending = toolCalls.get(index) ?? { id: "", name: "", inputJson: "" };
|
|
if (call?.id) pending.id = String(call.id);
|
|
if (call?.function?.name) pending.name = String(call.function.name);
|
|
if (typeof call?.function?.arguments === "string") pending.inputJson += call.function.arguments;
|
|
toolCalls.set(index, pending);
|
|
});
|
|
}
|
|
if (choice.finish_reason) finishReason = String(choice.finish_reason);
|
|
},
|
|
onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))),
|
|
onComplete: () => {
|
|
// Some compatible servers close after the finish chunk without [DONE].
|
|
if (finishReason) done();
|
|
else fail("OpenAI-compatible stream ended before the response completed");
|
|
},
|
|
onFailure: (message: string) => fail(`OpenAI-compatible connection failed: ${String(message)}`),
|
|
};
|
|
|
|
try {
|
|
request = openSseRequest(`${baseUrl}/chat/completions`, JSON.stringify(body), headers, listener);
|
|
} catch (error) {
|
|
setTimeout(() => fail(`OpenAI-compatible request failed: ${String((error as Error)?.message ?? error)}`), 0);
|
|
return { cancel: () => { settled = true; } };
|
|
}
|
|
|
|
return {
|
|
cancel: () => {
|
|
settled = true;
|
|
try { request?.cancel(); } catch { /* ignore */ }
|
|
},
|
|
};
|
|
}
|
|
|
|
/** String fields of a reasoning_details entry that stream in pieces. */
|
|
const REASONING_DETAIL_TEXT_FIELDS = ["text", "summary", "data"];
|
|
|
|
/**
|
|
* Rebuilds the complete reasoning_details array (OpenRouter's format) from
|
|
* streamed deltas. Each delta carries fragments; fragments with the same
|
|
* `index` belong to one entry, so their text, summary, and data concatenate
|
|
* and their other fields (e.g. a signature that arrives after the text) fill
|
|
* in. Entries without an index are kept as sent. The replayed array must
|
|
* match what the model produced, so it is reassembled whole rather than
|
|
* replaying only the latest chunk.
|
|
*/
|
|
class ReasoningDetailsAccumulator {
|
|
private readonly ordered: Record<string, any>[] = [];
|
|
private readonly byIndex = new Map<number, Record<string, any>>();
|
|
|
|
add(fragments: any[]): void {
|
|
for (const fragment of fragments) {
|
|
if (!fragment || typeof fragment !== "object" || Array.isArray(fragment)) continue;
|
|
const index = Number.isInteger(fragment.index) ? Number(fragment.index) : null;
|
|
const existing = index === null ? undefined : this.byIndex.get(index);
|
|
if (!existing) {
|
|
const entry = { ...fragment };
|
|
this.ordered.push(entry);
|
|
if (index !== null) this.byIndex.set(index, entry);
|
|
continue;
|
|
}
|
|
for (const [key, value] of Object.entries(fragment)) {
|
|
if (value === undefined || value === null) continue;
|
|
if (REASONING_DETAIL_TEXT_FIELDS.includes(key) && typeof value === "string" && typeof existing[key] === "string") {
|
|
existing[key] += value;
|
|
} else if (existing[key] === undefined || existing[key] === null) {
|
|
existing[key] = value;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
entries(): Record<string, any>[] {
|
|
return this.ordered;
|
|
}
|
|
}
|
|
|
|
/** Convert neutral conversation history to Chat Completions messages. */
|
|
function buildChatMessages(options: OpenAiCompatibleStreamOptions): any[] {
|
|
const messages: any[] = [];
|
|
if (options.system) messages.push({ role: "system", content: options.system });
|
|
|
|
for (const message of options.messages) {
|
|
if (typeof message.content === "string") {
|
|
messages.push({ role: message.role, content: message.content });
|
|
continue;
|
|
}
|
|
|
|
if (message.role === "assistant") {
|
|
let text = "";
|
|
const toolCalls: any[] = [];
|
|
let providerMetadata: Record<string, any> = {};
|
|
for (const block of message.content) {
|
|
if (block.type === "text") text += block.text;
|
|
else if (block.type === "tool_use") {
|
|
toolCalls.push({
|
|
id: block.id,
|
|
type: "function",
|
|
function: { name: block.name, arguments: JSON.stringify(block.input ?? {}) },
|
|
});
|
|
} else if (block.type === "provider_item" && block.provider === "openai-compatible") {
|
|
providerMetadata = { ...providerMetadata, ...block.item };
|
|
}
|
|
}
|
|
if (text || toolCalls.length) {
|
|
messages.push({
|
|
role: "assistant",
|
|
content: text || (toolCalls.length ? null : ""),
|
|
...(toolCalls.length ? { tool_calls: toolCalls } : {}),
|
|
...providerMetadata,
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
|
|
let pendingText = "";
|
|
const flushText = () => {
|
|
if (!pendingText) return;
|
|
messages.push({ role: "user", content: pendingText });
|
|
pendingText = "";
|
|
};
|
|
for (const block of message.content) {
|
|
if (block.type === "text") pendingText += block.text;
|
|
else if (block.type === "tool_result") {
|
|
flushText();
|
|
messages.push({
|
|
role: "tool",
|
|
tool_call_id: block.tool_use_id,
|
|
content: block.is_error ? `Error: ${block.content}` : block.content,
|
|
});
|
|
}
|
|
}
|
|
flushText();
|
|
}
|
|
return messages;
|
|
}
|
|
|
|
function describeHttpError(code: number, body: string): string {
|
|
let detail = "";
|
|
try {
|
|
const parsed = JSON.parse(body);
|
|
detail = String(parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? "");
|
|
} catch {
|
|
// Non-JSON error body.
|
|
}
|
|
switch (code) {
|
|
case 401: return "OpenAI-compatible: invalid API key";
|
|
case 403: return "OpenAI-compatible: API key lacks permission";
|
|
case 429: return "OpenAI-compatible: rate limited, try again shortly";
|
|
case 500:
|
|
case 502:
|
|
case 503:
|
|
case 504:
|
|
return "OpenAI-compatible: service unavailable, try again shortly";
|
|
default: return `OpenAI-compatible: HTTP ${code}${detail ? ` (${detail})` : ""}`;
|
|
}
|
|
}
|