fix streaming

server/src/llm/chat-tools.ts

@@ -853,6 +853,12 @@ function extractResponsesText(response: any, fallback = "") {
   return parts.join("") || fallback;
 }
 
+function getUnstreamedText(finalText: string, streamedText: string) {
+  if (!finalText) return "";
+  if (!streamedText) return finalText;
+  return finalText.startsWith(streamedText) ? finalText.slice(streamedText.length) : "";
+}
+
 function getResponseFailureMessage(response: any) {
   if (response?.status !== "failed" && response?.status !== "incomplete") return null;
   const errorMessage = typeof response?.error?.message === "string" ? response.error.message : null;
@@ -1113,6 +1119,9 @@ export async function* runToolAwareOpenAIChatStream(
     } as any);
 
     let roundText = "";
+    let streamedRoundText = "";
+    let roundHasToolCalls = false;
+    let canStreamRoundText = false;
     let completedResponse: any | null = null;
     const completedOutputItems: any[] = [];
 
@@ -1121,8 +1130,23 @@ export async function* runToolAwareOpenAIChatStream(
 
       if (event?.type === "response.output_text.delta" && typeof event.delta === "string") {
         roundText += event.delta;
+        if (canStreamRoundText && !roundHasToolCalls && event.delta.length) {
+          streamedRoundText += event.delta;
+          yield { type: "delta", text: event.delta };
+        }
+      } else if (event?.type === "response.output_item.added" && event.item) {
+        if (event.item.type === "function_call") {
+          roundHasToolCalls = true;
+          canStreamRoundText = false;
+        } else if (event.item.type === "message" && !roundHasToolCalls) {
+          canStreamRoundText = true;
+        }
       } else if (event?.type === "response.output_item.done" && event.item) {
         completedOutputItems[event.output_index ?? completedOutputItems.length] = event.item;
+        if (event.item.type === "function_call") {
+          roundHasToolCalls = true;
+          canStreamRoundText = false;
+        }
       } else if (event?.type === "response.completed") {
         completedResponse = event.response;
         sawUsage = mergeResponsesUsage(usageAcc, event.response?.usage) || sawUsage;
@@ -1144,13 +1168,18 @@ export async function* runToolAwareOpenAIChatStream(
     const normalizedToolCalls = normalizeResponsesToolCalls(responseOutputItems, round);
     if (!normalizedToolCalls.length) {
       const text = extractResponsesText(completedResponse, roundText);
-      if (danglingToolIntentRetries < MAX_DANGLING_TOOL_INTENT_RETRIES && looksLikeDanglingToolIntent(text)) {
+      if (
+        !streamedRoundText &&
+        danglingToolIntentRetries < MAX_DANGLING_TOOL_INTENT_RETRIES &&
+        looksLikeDanglingToolIntent(text)
+      ) {
         danglingToolIntentRetries += 1;
         appendDanglingToolIntentCorrection(input, text);
         continue;
       }
-      if (text) {
-        yield { type: "delta", text };
+      const unstreamedText = getUnstreamedText(text, streamedRoundText);
+      if (unstreamedText) {
+        yield { type: "delta", text: unstreamedText };
       }
       yield {
         type: "done",
@@ -1214,6 +1243,8 @@ export async function* runToolAwareChatCompletionsStream(
     } as any);
 
     let roundText = "";
+    let streamedRoundText = "";
+    let roundHasToolCalls = false;
     const roundToolCalls = new Map<number, { id?: string; name?: string; arguments: string }>();
 
     for await (const chunk of stream as any as AsyncIterable<any>) {
@@ -1224,9 +1255,16 @@ export async function* runToolAwareChatCompletionsStream(
       const deltaText = choice?.delta?.content ?? "";
       if (typeof deltaText === "string" && deltaText.length) {
         roundText += deltaText;
+        if (!roundHasToolCalls) {
+          streamedRoundText += deltaText;
+          yield { type: "delta", text: deltaText };
+        }
       }
 
       const deltaToolCalls = Array.isArray(choice?.delta?.tool_calls) ? choice.delta.tool_calls : [];
+      if (deltaToolCalls.length) {
+        roundHasToolCalls = true;
+      }
       for (const toolCall of deltaToolCalls) {
         const idx = typeof toolCall?.index === "number" ? toolCall.index : 0;
         const entry = roundToolCalls.get(idx) ?? { arguments: "" };
@@ -1252,13 +1290,18 @@ export async function* runToolAwareChatCompletionsStream(
       }));
 
     if (!normalizedToolCalls.length) {
-      if (danglingToolIntentRetries < MAX_DANGLING_TOOL_INTENT_RETRIES && looksLikeDanglingToolIntent(roundText)) {
+      if (
+        !streamedRoundText &&
+        danglingToolIntentRetries < MAX_DANGLING_TOOL_INTENT_RETRIES &&
+        looksLikeDanglingToolIntent(roundText)
+      ) {
         danglingToolIntentRetries += 1;
         appendDanglingToolIntentCorrection(conversation, roundText);
         continue;
       }
-      if (roundText) {
-        yield { type: "delta", text: roundText };
+      const unstreamedText = getUnstreamedText(roundText, streamedRoundText);
+      if (unstreamedText) {
+        yield { type: "delta", text: unstreamedText };
       }
       yield {
         type: "done",