Files
faceclaw/app/apps/translate/translate-llm.ts
T

84 lines
3.9 KiB
TypeScript

/**
* Streamed translation against Translate's OpenAI override, or the assistant's
* configured model when no override is set. Each call translates one stretch of speech into the selected output language; see
* TranslationModel for how stretches are batched and given context.
*/
import { streamAnthropicMessage } from "../../native/anthropic";
import { streamLocalModel } from "../../native/local-llm";
import { streamOpenAiResponse } from "../../native/openai";
import { streamOpenAiCompatibleChat } from "../../native/openai-chat";
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
import {
anthropicApiKeySetting, assistantModelSetting, assistantOpenAiCompatibleKeySetting,
assistantOpenAiCompatibleModelSetting, assistantOpenAiCompatibleUrlSetting, openAiApiKeySetting,
} from "../../ui/dashboard-settings";
import { buildTranslateSystemPrompt, buildTranslateUserMessage } from "../../prompts";
import { translateModelSetting, translateTargetLanguageSetting, spokenLanguageName } from "./translate-settings";
import { type TranslationTurn } from "./translation-model";
/**
* The model Translate uses, or null when none is configured. Hermes is an
* agent with its own tools and memory rather than a translator (every
* utterance would land in its session), so it falls back to Auto: the model
* for whichever API key is set, else the on-phone model.
*/
export function resolveTranslationModel(): ResolvedAssistantModel | null {
const translationModel = translateModelSetting.get().trim();
if (translationModel) {
const apiKey = openAiApiKeySetting.get().trim();
return apiKey ? { selection: "auto", provider: "openai", model: translationModel, apiKey } : null;
}
const selection = assistantModelSetting.get();
return resolveAssistantModel(selection === "hermes" ? "auto" : selection, {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
});
}
export type TranslationRequest = {
speech: string;
context: TranslationTurn[];
/** English name of the spoken language when the user set one, else "". */
language: string;
onProgress: (textSoFar: string) => void;
onDone: (translation: string) => void;
onError: (message: string) => void;
};
export function streamTranslation(model: ResolvedAssistantModel, request: TranslationRequest): LlmStreamHandle {
const targetLanguage = spokenLanguageName(translateTargetLanguageSetting.get());
const options: LlmStreamOptions = {
apiKey: model.apiKey,
model: model.model,
system: buildTranslateSystemPrompt(targetLanguage),
effort: model.effort,
maxTokens: 1024,
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context, request.language, targetLanguage) }],
onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)),
onDone: (result) => {
if (result.stopReason === "refusal") request.onError("The model declined to translate this.");
else request.onDone(cleanTranslation(result.text));
},
onError: request.onError,
};
switch (model.provider) {
case "openai":
return streamOpenAiResponse(options);
case "openai-compatible":
return streamOpenAiCompatibleChat({ ...options, baseUrl: model.baseUrl ?? "" });
case "local":
return streamLocalModel(options);
default:
return streamAnthropicMessage(options);
}
}
/** The context turns are tagged <translation>; a model may echo the tag around its reply. */
function cleanTranslation(text: string): string {
return text.replace(/^\s*<(?:translation|english)>/i, "").replace(/<\/(?:translation|english)>\s*$/i, "").trim();
}