forked from Blendman974/faceclaw
Add support for a local model (Qwen3)
This commit is contained in:
1 parent
6103c00ca3
commit
6f45888627
16 files changed
+1532
-15
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import { streamAnthropicMessage } from "../native/anthropic";
|
||||
import { streamLocalQwen } from "../native/llama";
|
||||
import { streamOpenAiResponse } from "../native/openai";
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
@@ -20,6 +21,8 @@ import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
|
||||
|
||||
/** Safety caps so a tool-thrashing model can't loop or run forever. */
|
||||
const MAX_ITERATIONS = 8;
|
||||
/** The on-phone model is slower per step and unreliable on long tool chains. */
|
||||
const MAX_ITERATIONS_LOCAL = 4;
|
||||
const MAX_TURN_MS = 2 * 60 * 1000;
|
||||
|
||||
export type DirectTurnOptions = {
|
||||
@@ -54,9 +57,10 @@ export class DirectAssistantBackend {
|
||||
options.callbacks.onError(message);
|
||||
};
|
||||
|
||||
const maxIterations = options.provider === "local" ? MAX_ITERATIONS_LOCAL : MAX_ITERATIONS;
|
||||
const runIteration = (iteration: number): void => {
|
||||
if (cancelled) return;
|
||||
if (iteration >= MAX_ITERATIONS) {
|
||||
if (iteration >= maxIterations) {
|
||||
finishError("Assistant stopped: too many tool steps");
|
||||
return;
|
||||
}
|
||||
@@ -115,9 +119,12 @@ export class DirectAssistantBackend {
|
||||
finishError(message);
|
||||
},
|
||||
};
|
||||
streamHandle = options.provider === "openai"
|
||||
? streamOpenAiResponse(streamOptions)
|
||||
: streamAnthropicMessage(streamOptions);
|
||||
streamHandle =
|
||||
options.provider === "openai"
|
||||
? streamOpenAiResponse(streamOptions)
|
||||
: options.provider === "local"
|
||||
? streamLocalQwen(streamOptions)
|
||||
: streamAnthropicMessage(streamOptions);
|
||||
};
|
||||
|
||||
runIteration(0);
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
import type { LlmToolDefinition } from "./llm-protocol";
|
||||
|
||||
/**
|
||||
* Generate a GBNF grammar (llama.cpp grammar dialect) that constrains a
|
||||
* Qwen-style tool call to be well-formed: one <tool_call> block whose JSON
|
||||
* names a real tool and whose arguments match that tool's schema. Used as a
|
||||
* *lazy* grammar — sampling is unconstrained until the model emits
|
||||
* <tool_call>, so plain text answers are unaffected, and once the block
|
||||
* closes the grammar is complete (the model can only end the message, which
|
||||
* serializes multi-call turns into one call per loop iteration — deliberate:
|
||||
* small models are far more reliable one call at a time).
|
||||
*
|
||||
* Our tool schemas are flat objects of string/number/boolean/enum properties.
|
||||
* Anything fancier degrades to a generic JSON value rule rather than failing.
|
||||
*/
|
||||
|
||||
type JsonSchema = {
|
||||
type?: string;
|
||||
enum?: unknown[];
|
||||
properties?: Record<string, JsonSchema>;
|
||||
required?: string[];
|
||||
description?: string;
|
||||
};
|
||||
|
||||
/** Longest run of optional properties we'll enumerate subsets for. */
|
||||
const MAX_SUBSET_PROPS = 5;
|
||||
|
||||
export function buildToolCallGrammar(tools: LlmToolDefinition[]): string | null {
|
||||
if (!tools.length) return null;
|
||||
const used = new Set<string>();
|
||||
const toolRules: string[] = [];
|
||||
for (let i = 0; i < tools.length; i++) {
|
||||
const args = argsRule(tools[i]!.input_schema as JsonSchema, used);
|
||||
toolRules.push(
|
||||
`tc${i} ::= "<tool_call>" nl "{" sp ${jsonKey("name")} sp ":" sp "\\"${gbnfEscape(
|
||||
tools[i]!.name,
|
||||
)}\\"" sp "," sp ${jsonKey("arguments")} sp ":" sp ${args} sp "}" nl "</tool_call>"`,
|
||||
);
|
||||
}
|
||||
used.add("sp");
|
||||
used.add("nl");
|
||||
|
||||
const lines = [`root ::= ${toolRules.map((_, i) => `tc${i}`).join(" | ")}`, ...toolRules];
|
||||
for (const [name, rule] of SHARED_RULES) {
|
||||
if (used.has(name)) lines.push(rule);
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** Build the arguments-object expression for one tool schema. */
|
||||
function argsRule(schema: JsonSchema | undefined, used: Set<string>): string {
|
||||
const properties = schema?.properties ?? {};
|
||||
const names = Object.keys(properties);
|
||||
if (!names.length) return `"{" sp "}"`;
|
||||
const required = new Set(schema?.required ?? []);
|
||||
const optionalCount = names.filter((n) => !required.has(n)).length;
|
||||
if (names.length > MAX_SUBSET_PROPS && optionalCount > 0) {
|
||||
used.add("jobj");
|
||||
return "jobj";
|
||||
}
|
||||
|
||||
// Enumerate every subset of properties that includes all required ones,
|
||||
// keeping declaration order. Small N keeps this tiny (<= 2^5 alternatives).
|
||||
const members = names.map((name) => {
|
||||
const value = valueExpr(properties[name]!, used);
|
||||
return `${jsonKey(name)} sp ":" sp ${value}`;
|
||||
});
|
||||
const alternatives: string[] = [];
|
||||
for (let mask = 0; mask < 1 << names.length; mask++) {
|
||||
const chosen: string[] = [];
|
||||
let valid = true;
|
||||
for (let j = 0; j < names.length; j++) {
|
||||
const included = (mask & (1 << j)) !== 0;
|
||||
if (included) chosen.push(members[j]!);
|
||||
else if (required.has(names[j]!)) valid = false;
|
||||
}
|
||||
if (!valid) continue;
|
||||
if (!chosen.length) {
|
||||
alternatives.push(`"{" sp "}"`);
|
||||
} else {
|
||||
alternatives.push(`"{" sp ${chosen.join(` sp "," sp `)} sp "}"`);
|
||||
}
|
||||
}
|
||||
return alternatives.length === 1 ? alternatives[0]! : `(${alternatives.join(" | ")})`;
|
||||
}
|
||||
|
||||
/** Grammar expression for one property value. */
|
||||
function valueExpr(schema: JsonSchema, used: Set<string>): string {
|
||||
if (Array.isArray(schema.enum) && schema.enum.length) {
|
||||
if (schema.enum.every((v) => typeof v === "string")) {
|
||||
return `(${schema.enum.map((v) => `"\\"${gbnfEscape(String(v))}\\""`).join(" | ")})`;
|
||||
}
|
||||
if (schema.enum.every((v) => typeof v === "number" || typeof v === "boolean")) {
|
||||
return `(${schema.enum.map((v) => `"${String(v)}"`).join(" | ")})`;
|
||||
}
|
||||
}
|
||||
switch (schema.type) {
|
||||
case "string":
|
||||
used.add("jstring");
|
||||
used.add("jchar");
|
||||
used.add("jhex");
|
||||
return "jstring";
|
||||
case "number":
|
||||
used.add("jnumber");
|
||||
return "jnumber";
|
||||
case "integer":
|
||||
used.add("jint");
|
||||
return "jint";
|
||||
case "boolean":
|
||||
used.add("jbool");
|
||||
return "jbool";
|
||||
default:
|
||||
for (const name of ["jvalue", "jstring", "jchar", "jhex", "jnumber", "jbool", "jarray", "jobj", "jmember"]) {
|
||||
used.add(name);
|
||||
}
|
||||
return "jvalue";
|
||||
}
|
||||
}
|
||||
|
||||
/** A JSON object key as a GBNF literal: "name" -> "\"name\"". */
|
||||
function jsonKey(name: string): string {
|
||||
return `"\\"${gbnfEscape(name)}\\""`;
|
||||
}
|
||||
|
||||
/** Escape for the inside of a GBNF double-quoted literal. */
|
||||
function gbnfEscape(text: string): string {
|
||||
return text.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, "\\n");
|
||||
}
|
||||
|
||||
// Terminal rules, emitted only when referenced. jchar mirrors llama.cpp's
|
||||
// json.gbnf string character rule.
|
||||
const SHARED_RULES: Array<[string, string]> = [
|
||||
["sp", `sp ::= " "?`],
|
||||
["nl", `nl ::= "\\n"`],
|
||||
["jstring", `jstring ::= "\\"" jchar* "\\""`],
|
||||
["jchar", `jchar ::= [^"\\\\\\x7F\\x00-\\x1F] | "\\\\" (["\\\\bfnrt] | "u" jhex jhex jhex jhex)`],
|
||||
["jhex", `jhex ::= [0-9a-fA-F]`],
|
||||
["jnumber", `jnumber ::= "-"? ("0" | [1-9] [0-9]*) ("." [0-9]+)? (("e" | "E") ("-" | "+")? [0-9]+)?`],
|
||||
["jint", `jint ::= "-"? ("0" | [1-9] [0-9]*)`],
|
||||
["jbool", `jbool ::= "true" | "false"`],
|
||||
["jvalue", `jvalue ::= jstring | jnumber | jbool | "null" | jarray | jobj`],
|
||||
["jarray", `jarray ::= "[" sp (jvalue (sp "," sp jvalue)*)? sp "]"`],
|
||||
["jobj", `jobj ::= "{" sp (jmember (sp "," sp jmember)*)? sp "}"`],
|
||||
["jmember", `jmember ::= jstring sp ":" sp jvalue`],
|
||||
];
|
||||
@@ -0,0 +1,81 @@
|
||||
import type { LlmContentBlock, LlmMessage, LlmToolDefinition } from "./llm-protocol";
|
||||
|
||||
/**
|
||||
* Render our provider-neutral message history into a Qwen3 (ChatML) prompt
|
||||
* string, matching the model's chat template closely enough that its
|
||||
* tool-calling priors apply: Hermes-style <tools> declarations in the system
|
||||
* block, <tool_call> JSON blocks in assistant turns, and tool results wrapped
|
||||
* in <tool_response> inside a user turn. The caller appends nothing — the
|
||||
* returned string ends with the assistant generation header.
|
||||
*/
|
||||
|
||||
const IM_START = "<|im_start|>";
|
||||
const IM_END = "<|im_end|>";
|
||||
|
||||
export function buildQwenPrompt(
|
||||
system: string | undefined,
|
||||
messages: LlmMessage[],
|
||||
tools: LlmToolDefinition[] | undefined,
|
||||
): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`${IM_START}system\n${systemBlock(system, tools)}${IM_END}\n`);
|
||||
for (const message of messages) {
|
||||
if (message.role === "assistant") {
|
||||
parts.push(`${IM_START}assistant\n${assistantBlock(message.content)}${IM_END}\n`);
|
||||
} else {
|
||||
parts.push(`${IM_START}user\n${userBlock(message.content)}${IM_END}\n`);
|
||||
}
|
||||
}
|
||||
parts.push(`${IM_START}assistant\n`);
|
||||
return parts.join("");
|
||||
}
|
||||
|
||||
function systemBlock(system: string | undefined, tools: LlmToolDefinition[] | undefined): string {
|
||||
const base = system?.trim() ?? "You are a helpful assistant.";
|
||||
if (!tools?.length) return base;
|
||||
const declarations = tools
|
||||
.map((tool) =>
|
||||
JSON.stringify({
|
||||
type: "function",
|
||||
function: { name: tool.name, description: tool.description, parameters: tool.input_schema },
|
||||
}),
|
||||
)
|
||||
.join("\n");
|
||||
return (
|
||||
`${base}\n\n# Tools\n\n` +
|
||||
`You may call one or more functions to assist with the user query.\n\n` +
|
||||
`You are provided with function signatures within <tools></tools> XML tags:\n` +
|
||||
`<tools>\n${declarations}\n</tools>\n\n` +
|
||||
`For each function call, return a json object with function name and arguments ` +
|
||||
`within <tool_call></tool_call> XML tags:\n` +
|
||||
`<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>`
|
||||
);
|
||||
}
|
||||
|
||||
function assistantBlock(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content;
|
||||
const parts: string[] = [];
|
||||
for (const block of content) {
|
||||
if (block.type === "text") {
|
||||
if (block.text.trim()) parts.push(block.text.trim());
|
||||
} else if (block.type === "tool_use") {
|
||||
parts.push(`<tool_call>\n${JSON.stringify({ name: block.name, arguments: block.input ?? {} })}\n</tool_call>`);
|
||||
}
|
||||
// provider_item blocks are OpenAI-specific continuation state; skip.
|
||||
}
|
||||
return parts.join("\n");
|
||||
}
|
||||
|
||||
function userBlock(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content;
|
||||
const parts: string[] = [];
|
||||
for (const block of content) {
|
||||
if (block.type === "text") {
|
||||
if (block.text.trim()) parts.push(block.text.trim());
|
||||
} else if (block.type === "tool_result") {
|
||||
const body = block.is_error ? `Error: ${block.content}` : block.content || "(no output)";
|
||||
parts.push(`<tool_response>\n${body}\n</tool_response>`);
|
||||
}
|
||||
}
|
||||
return parts.join("\n");
|
||||
}
|
||||
@@ -1,4 +1,8 @@
|
||||
import { buildAssistantSystemPrompt } from "../prompts";
|
||||
import {
|
||||
ASSISTANT_SYSTEM_PROMPT_BASE,
|
||||
buildAssistantSystemPrompt,
|
||||
describeAssistantContext,
|
||||
} from "../prompts";
|
||||
import { assistantBridge } from "./bridge-client";
|
||||
import { DirectAssistantBackend } from "./direct-backend";
|
||||
import type { LlmMessage, LlmToolDefinition } from "./llm-protocol";
|
||||
@@ -108,15 +112,23 @@ export class AssistantSession {
|
||||
return;
|
||||
}
|
||||
|
||||
this.messages.push({ role: "user", content: text });
|
||||
this.trimHistory();
|
||||
const llm = this.config.llm;
|
||||
// Local provider: keep the system prompt (and thus the tool declarations
|
||||
// that follow it in the rendered prompt) byte-stable so the on-phone
|
||||
// model's KV prefix cache survives across turns; the volatile context
|
||||
// (time, foreground app, battery) rides along with the utterance instead.
|
||||
const isLocal = llm.provider === "local";
|
||||
this.messages.push({
|
||||
role: "user",
|
||||
content: isLocal ? `${text}\n\n(${describeAssistantContext(ctx)})` : text,
|
||||
});
|
||||
this.trimHistory();
|
||||
this.turnHandle = this.directBackend.runTurn({
|
||||
provider: llm.provider,
|
||||
apiKey: llm.apiKey,
|
||||
model: llm.model,
|
||||
effort: llm.effort,
|
||||
system: buildAssistantSystemPrompt(ctx),
|
||||
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
|
||||
messages: this.messages,
|
||||
buildTools: () => this.buildToolDefinitions(),
|
||||
registry: this.registry,
|
||||
|
||||
Reference in new issue
Block a user