Files
faceclaw/app/apps/microphones/speaker-insights.ts

184 lines
6.9 KiB
TypeScript

import { Utils } from "@nativescript/core";
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
import { speakerRegistry } from "./speakers";
declare const com: any;
/**
* Per-speaker conversation insights, generated with the on-phone LLM when a
* caption session ends: a short recap of the conversation, the action items
* it produced, and a consolidated list of inferred facts about the person
* (fed by the transcript, the contact's tag/notes, and previously inferred
* facts). Stored on the speaker row in the conversation store, where the
* encounter popup and the People view read them back.
*
* Best-effort by design: no model, a busy runner, or an unparseable reply
* simply leaves the speaker's existing insights in place. Everything runs
* sequentially — the runner is shared with the assistant and caption name
* verification.
*/
/** Speakers with fewer transcript lines than this in the session are skipped. */
const MIN_SPEAKER_LINES = 2;
/** Transcript excerpt budget; Qwen runs with a 4096-token context. */
const MAX_TRANSCRIPT_CHARS = 4500;
const MAX_FACTS = 8;
const MAX_ACTION_ITEMS = 6;
const GENERATION_TIMEOUT_MS = 90_000;
type SegmentLine = { speakerId: number | null; text: string };
function store(): any {
const context = Utils.android.getApplicationContext();
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
}
/** One generation at a time across sessions; a second session-end call queues behind the first. */
let chain: Promise<void> = Promise.resolve();
/**
* Generate and store insights for every non-wearer speaker heard in the
* session. Fire-and-forget: call after the session row is finished; errors
* only log.
*/
export function generateSessionInsights(sessionId: number): void {
if (!global.isAndroid || sessionId < 0 || !isLocalModelReady()) return;
chain = chain.then(() => generateForSession(sessionId)).catch((error) => {
console.warn(`speaker insights generation failed: ${error}`);
});
}
async function generateForSession(sessionId: number): Promise<void> {
const segments = (JSON.parse(String(store().querySegments(sessionId))) as Array<Record<string, unknown>>)
.map((row): SegmentLine => ({
speakerId: row.speakerId === null ? null : Number(row.speakerId),
text: String(row.text ?? "").trim(),
}))
.filter((line) => line.text);
if (!segments.length) return;
speakerRegistry.reload();
const lineCounts = new Map<number, number>();
for (const line of segments) {
if (line.speakerId !== null) {
lineCounts.set(line.speakerId, (lineCounts.get(line.speakerId) ?? 0) + 1);
}
}
const transcript = buildTranscript(segments);
for (const [speakerId, lines] of lineCounts) {
if (lines < MIN_SPEAKER_LINES) continue;
const profile = speakerRegistry.byId(speakerId);
if (!profile || profile.isWearer) continue;
try {
const insights = await generateForSpeaker(profile.name, profile.tag, profile.facts, transcript);
if (insights) {
speakerRegistry.setInsights(
speakerId,
insights.recap,
insights.actionItems.slice(0, MAX_ACTION_ITEMS),
insights.facts.slice(0, MAX_FACTS),
sessionId,
);
}
} catch (error) {
console.warn(`speaker insights failed for ${profile.name}: ${error}`);
}
}
}
function buildTranscript(segments: SegmentLine[]): string {
const names = new Map<number, string>();
for (const profile of speakerRegistry.all()) {
names.set(profile.id, profile.isWearer ? `${profile.name} (the wearer)` : profile.name);
}
const lines = segments.map((line) => {
const who = line.speakerId !== null ? (names.get(line.speakerId) ?? "Speaker") : "Speaker";
return `${who}: ${line.text}`;
});
let text = lines.join("\n");
if (text.length > MAX_TRANSCRIPT_CHARS) {
// Keep the end of the conversation: goodbyes and plans (the action items)
// cluster there, and the recap should reflect where things landed.
text = "..." + text.slice(text.length - MAX_TRANSCRIPT_CHARS);
}
return text;
}
type SpeakerInsights = { recap: string; actionItems: string[]; facts: string[] };
function generateForSpeaker(
name: string,
notes: string,
existingFacts: string[],
transcript: string,
): Promise<SpeakerInsights | null> {
return new Promise((resolve) => {
let settled = false;
const finish = (value: SpeakerInsights | null) => {
if (!settled) {
settled = true;
resolve(value);
}
};
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
try {
streamLocalModel({
apiKey: "",
model: "local",
system:
"You maintain a contact's conversation memory for a live-captioning app. " +
"From the transcript, produce for the named person: a recap of this conversation " +
"(1-2 sentences), the concrete action items agreed or requested in it (things to do, " +
"send, or follow up on — empty if none), and an updated list of short facts about the " +
"person (job, family, interests, plans, preferences). Merge the previously known facts " +
"with anything new from the transcript; drop duplicates and anything now outdated. " +
"Never invent details that are not supported by the notes, known facts, or transcript. " +
'Reply with ONLY a JSON object: {"recap": "...", "actionItems": ["..."], "facts": ["..."]}.',
messages: [
{
role: "user",
content:
`Person: ${name}\n` +
(notes ? `Contact notes: ${notes}\n` : "") +
`Previously known facts: ${existingFacts.length ? JSON.stringify(existingFacts) : "(none)"}\n\n` +
`# Transcript\n${transcript}`,
},
],
maxTokens: 350,
onDone: (result) => {
clearTimeout(timer);
finish(parseInsights(result.text));
},
onError: (message) => {
clearTimeout(timer);
console.warn(`speaker insights model error: ${message}`);
finish(null);
},
});
} catch (error) {
clearTimeout(timer);
console.warn(`speaker insights request failed: ${error}`);
finish(null);
}
});
}
/** Parse the model's reply leniently: the first {...} block wins. */
export function parseInsights(reply: string): SpeakerInsights | null {
const start = reply.indexOf("{");
const end = reply.lastIndexOf("}");
if (start < 0 || end <= start) return null;
try {
const parsed = JSON.parse(reply.slice(start, end + 1)) as Record<string, unknown>;
const recap = String(parsed.recap ?? "").trim();
if (!recap) return null;
const list = (value: unknown): string[] =>
Array.isArray(value) ? value.map((item) => String(item).trim()).filter(Boolean) : [];
return { recap, actionItems: list(parsed.actionItems), facts: list(parsed.facts) };
} catch {
return null;
}
}