What is a context window? Giving a café chatbot a memory

const messages: Anthropic.MessageParam[] = [{ role: "user", content: question }];

05-conversation-memory/
├── package.json
└── src/
    ├── tools.ts           # unchanged
    ├── call-llm.ts        # unchanged
    ├── prompts.ts         # unchanged
    ├── shop-info.ts       # unchanged
    ├── orders.ts          # adds a resetOrders function
    ├── offline.ts         # scripted model, now reads the history too
    ├── agent.ts           # the loop, now takes and returns the history
    ├── session.ts         # NEW: one conversation with one customer
    ├── trim-history.ts    # NEW: two ways to trim the history, and a checker
    ├── memory.ts          # NEW: remembers which order the customer means
    ├── two-messages.ts    # run 1
    ├── morning.ts         # run 2
    ├── bad-trim.ts        # run 3
    └── forgot-order.ts    # run 4

import type Anthropic from "@anthropic-ai/sdk";

import { callModel, textOf } from "./call-llm.ts";
import { SYSTEM_V4 } from "./prompts.ts";
import { runTool } from "./tools.ts";

export type Message = Anthropic.MessageParam;
export type ModelCall = { messageCount: number; chars: number; inputTokens: number };
export type AgentResult = { answer: string; history: Message[]; log: string[]; calls: ModelCall[] };

// The agent loop from the previous post, with one difference: it takes the conversation history and returns the new one.
export async function runAgent(history: Message[], question: string, system = SYSTEM_V4, maxCalls = 6): Promise<AgentResult> {
  const messages: Message[] = [...history, { role: "user", content: question }];
  const log: string[] = [];
  const calls: ModelCall[] = [];

  for (let call = 1; call <= maxCalls; call++) {
    const reply = await callModel(system, messages);
    // Record how much each call sent: messages, characters, and tokens when calling the real API.
    calls.push({ messageCount: messages.length, chars: JSON.stringify(messages).length, inputTokens: reply.inputTokens });
    messages.push({ role: "assistant", content: reply.content });

    if (reply.stop_reason !== "tool_use") return { answer: textOf(reply.content), history: messages, log, calls };

    const results: Anthropic.ToolResultBlockParam[] = [];
    for (const block of reply.content) {
      if (block.type !== "tool_use") continue;
      const result = runTool(block.name, block.input);
      log.push(`${block.name}(${JSON.stringify(block.input)}) → ${result.content}${result.isError ? "  (is_error)" : ""}`);
      results.push({ type: "tool_result", tool_use_id: block.id, content: result.content, is_error: result.isError });
    }
    messages.push({ role: "user", content: results });
  }

  const apology = "Sorry, Mộc couldn't finish this request. A staff member will check it and message you shortly.";
  messages.push({ role: "assistant", content: apology });
  return { answer: apology, history: messages, log: [...log, `stopped: called the model ${maxCalls} times`], calls };
}

import { runAgent, type AgentResult, type Message } from "./agent.ts";
import { SYSTEM_V4 } from "./prompts.ts";

export type SessionOptions = {
  // Choose which part of the history to send to the model. Default: all of it.
  pickHistory?: (history: Message[]) => Message[];
  // Build the system prompt for this call from the full history. Default: the shop's fixed prompt.
  buildSystem?: (history: Message[]) => string;
};

// A session is one conversation with one customer. It keeps the full history in the process's memory.
export function createSession({ pickHistory = (history) => history, buildSystem = () => SYSTEM_V4 }: SessionOptions = {}) {
  let history: Message[] = [];
  return {
    async receive(message: string): Promise<AgentResult> {
      const sent = pickHistory(history);
      const result = await runAgent(sent, message, buildSystem(history));
      // This turn's new messages are the ones after the part that was sent; append them to the FULL history.
      history = [...history, ...result.history.slice(sent.length)];
      return result;
    },
    get history() {
      return history;
    },
  };
}

// The question of the current turn: the customer's latest text message.
function currentQuestion(messages: Anthropic.MessageParam[]): { question: string; index: number } {
  const index = messages.findLastIndex((message) => message.role === "user" && typeof message.content === "string");
  return { question: String(messages[index].content).toLowerCase(), index };
}

// Imitates a real model reading back through the conversation: look for an order number in the <memory> tag
// of the system prompt, then in the messages and tool results before the question, nearest first.
function orderMentioned(system: string, messages: Anthropic.MessageParam[], before: number): string | undefined {
  const remembered = /<memory>[\s\S]*?order (\d{4})/.exec(system)?.[1];
  if (remembered) return remembered;
  for (let i = before - 1; i >= 0; i--) {
    const id = /\b(\d{4})\b/.exec(JSON.stringify(messages[i].content))?.[1];
    if (id) return id;
  }
  return undefined;
}

export function offlineModel(system: string, messages: Anthropic.MessageParam[]): Reply {
  const { question, index } = currentQuestion(messages); // previous post: messages[0]
  const last = messages.at(-1)!;

  // A new turn: the last message is the customer's question.
  if (typeof last.content === "string") {
    const id = question.match(/\b\d{4}\b/)?.[0];
    if (id && wantsToAdd(question)) return callTool("add_item", { order_id: id, item: itemIn(question)!, quantity: 1 });
    if (id) return callTool("get_order", { order_id: id });
    const phone = question.match(/\b0\d{9}\b/)?.[0];
    if (phone) return callTool("find_order_by_phone", { phone });
    if (wantsToAdd(question)) {
      // NEW: no order number given, so look in the earlier part of the conversation.
      const earlier = orderMentioned(system, messages, index);
      if (earlier) return callTool("add_item", { order_id: earlier, item: itemIn(question)!, quantity: 1 });
      return answer("Could you send Mộc your order number or the phone number you ordered with, so we can add it?");
    }
    if (/what time|open/.test(question)) return answer(`Hi! ${system.split("\n").find((line) => /is open from/.test(line)) ?? ""}`);
    // NEW: two more kinds of questions, so the long conversation later has something to talk about.
    if (/deliver|shipping/.test(question)) return answer(system.split("\n").find((line) => /delivers within/.test(line)) ?? "");
    const item = itemIn(question);
    if (item) return answer(`A ${item} at Mộc is ${MENU[item].toLocaleString("en-US")} đồng.`);
    return answer("Sorry, Mộc doesn't have that information.");
  }

  // Mid-turn: the tool-result handling is the same as in the previous post.
  // ...
}

// Two messages in a row from the same customer, this time through a session that keeps the history.
import { createSession } from "./session.ts";

const session = createSession();
for (const message of ["Has my order 1041 been delivered yet?", "Then please add a bac xiu as well"]) {
  const result = await session.receive(message);
  console.log(`Customer: ${message}`);
  for (const line of result.log) console.log(`  ${line}`);
  console.log(`Mộc:      ${result.answer}`);
  console.log(`          (the history now has ${session.history.length} messages)\n`);
}

Customer: Has my order 1041 been delivered yet?
  get_order({"order_id":"1041"}) → {"order_id":"1041","items":["milk coffee","egg banh mi"],"status":"being made","placed_at":"today 8:12"}
Mộc:      Your order 1041 (milk coffee, egg banh mi), placed today 8:12, is being made.
          (the history now has 4 messages)

Customer: Then please add a bac xiu as well
  add_item({"order_id":"1041","item":"bac xiu","quantity":1}) → {"order_id":"1041","items":["milk coffee","egg banh mi","bac xiu"],"status":"being made"}
Mộc:      Done! Order 1041 now has: milk coffee, egg banh mi, bac xiu, and it is still being made.
          (the history now has 8 messages)

// One customer sends 30 messages over the morning. After each one, print how much the last model call sent.
import { createSession } from "./session.ts";

const QUESTIONS = [
  "Has my order 1041 been delivered yet?",
  "What time do you close?",
  "How much is a salt coffee?",
  "How much is delivery?",
  "What does an egg banh mi cost?",
  "How much is a bac xiu?",
];

const session = createSession();
let totalChars = 0;
let totalCalls = 0;
console.log("message | messages sent | chars sent | running total chars");
for (let i = 1; i <= 30; i++) {
  const result = await session.receive(QUESTIONS[(i - 1) % QUESTIONS.length]);
  totalCalls += result.calls.length;
  for (const call of result.calls) totalChars += call.chars;
  const last = result.calls.at(-1)!;
  if (i === 1 || i % 5 === 0) {
    console.log(`${String(i).padStart(7)} | ${String(last.messageCount).padStart(13)} | ${String(last.chars).padStart(10)} | ${String(totalChars).padStart(19)}`);
  }
}
console.log(`\n30 customer messages, ${totalCalls} model calls, the final history has ${session.history.length} messages.`);

message | messages sent | chars sent | running total chars
      1 |             3 |        425 |                 492
      5 |            11 |       1125 |                3933
     10 |            23 |       2285 |               14817
     15 |            35 |       3501 |               32707
     20 |            47 |       4696 |               57547
     25 |            59 |       5869 |               89358
     30 |            69 |       6715 |              121290

30 customer messages, 35 model calls, the final history has 70 messages.

import type Anthropic from "@anthropic-ai/sdk";

import type { Message } from "./agent.ts";

// The naive way: keep the last N messages, whatever they are.
export function keepLastMessages(history: Message[], keep: number): Message[] {
  return history.slice(-keep);
}

// A turn starts with a customer message (role user, text content) and runs until the next customer message.
const startsTurn = (message: Message) => message.role === "user" && typeof message.content === "string";

// The right way: keep the last N turns. It always cuts at a turn boundary, so it never splits a tool_use / tool_result pair.
export function keepLastTurns(history: Message[], turns: number): Message[] {
  const starts = history.flatMap((message, i) => (startsTurn(message) ? [i] : []));
  if (starts.length <= turns) return history;
  return history.slice(starts[starts.length - turns]);
}

// Checks two Messages API rules that a badly trimmed history breaks. Returns the first problem, or null.
// I wrote this to catch the problem offline; the real API answers with a 400 error in its own words.
export function checkHistory(messages: Message[]): string | null {
  if (messages[0]?.role !== "user") return `the first message has the role "${messages[0]?.role}", it must be "user"`;
  for (let i = 0; i < messages.length; i++) {
    const content = messages[i].content;
    if (typeof content === "string") continue;
    for (const block of content) {
      if (block.type !== "tool_result") continue;
      const before = i > 0 ? messages[i - 1].content : [];
      const paired = typeof before !== "string" && before.some((b) => b.type === "tool_use" && (b as Anthropic.ToolUseBlock).id === block.tool_use_id);
      if (!paired) return `message ${i} holds the tool_result for ${block.tool_use_id}, but the message right before it has no such tool_use`;
    }
  }
  return null;
}

// Trim the history by message count, then check whether what's left can be sent to the API.
import type { Message } from "./agent.ts";
import { createSession } from "./session.ts";
import { checkHistory, keepLastMessages, keepLastTurns } from "./trim-history.ts";

const session = createSession();
for (const message of ["I ordered this morning, my number is 0901000001", "What time do you close?", "How much is delivery?"]) {
  await session.receive(message);
}

const describe = (message: Message) =>
  typeof message.content === "string" ? `"${message.content}"` : message.content.map((b) => b.type).join(" + ");

console.log(`Full history (${session.history.length} messages):`);
session.history.forEach((message, i) => console.log(`  ${i}  ${message.role.padEnd(9)} ${describe(message)}`));

for (const keep of [8, 7, 6]) {
  const trimmed = keepLastMessages(session.history, keep);
  console.log(`\nKeep the last ${keep} messages → ${checkHistory(trimmed) ?? "valid"}`);
}
const byTurn = keepLastTurns(session.history, 2);
console.log(`\nKeep the last 2 turns (${byTurn.length} messages) → ${checkHistory(byTurn) ?? "valid"}`);

Full history (10 messages):
  0  user      "I ordered this morning, my number is 0901000001"
  1  assistant tool_use
  2  user      tool_result
  3  assistant tool_use
  4  user      tool_result
  5  assistant text
  6  user      "What time do you close?"
  7  assistant text
  8  user      "How much is delivery?"
  9  assistant text

Keep the last 8 messages → message 0 holds the tool_result for toolu_offline_1, but the message right before it has no such tool_use

Keep the last 7 messages → the first message has the role "assistant", it must be "user"

Keep the last 6 messages → message 0 holds the tool_result for toolu_offline_2, but the message right before it has no such tool_use

Keep the last 2 turns (4 messages) → valid

const INITIAL_ORDERS: Order[] = [
  { orderId: "1041", phone: "0901000001", items: ["milk coffee", "egg banh mi"], status: "being made", placedAt: "today 8:12" },
  { orderId: "1042", phone: "0901000002", items: ["bac xiu"], status: "out for delivery", placedAt: "today 9:40" },
];

let ORDERS: Order[] = structuredClone(INITIAL_ORDERS);

// Put the data back the way it started, so test runs don't affect each other.
export function resetOrders(): void {
  ORDERS = structuredClone(INITIAL_ORDERS);
}

import type { Message } from "./agent.ts";
import { SYSTEM_V4 } from "./prompts.ts";

// Read the full history and pull out what needs remembering: the latest order number a tool returned.
// The code extracts it from tool results itself instead of asking the model to summarize, so it can't remember it wrong.
export function currentOrder(history: Message[]): string | null {
  let orderId: string | null = null;
  for (const message of history) {
    if (typeof message.content === "string") continue;
    for (const block of message.content) {
      if (block.type !== "tool_result" || block.is_error) continue;
      const id = /"order_id":"(\d{4})"/.exec(String(block.content))?.[1];
      if (id) orderId = id;
    }
  }
  return orderId;
}

// The shop's system prompt, plus a <memory> tag once we know which order the customer is talking about.
export function systemWithMemory(history: Message[]): string {
  const id = currentOrder(history);
  if (!id) return SYSTEM_V4;
  return `${SYSTEM_V4}\n\n<memory>\nIn this conversation the customer is talking about order ${id}.\n</memory>`;
}

// The customer gives the order number early, asks a few other things, then asks to add an item.
// Run it three times with three ways of choosing the history, to compare.
import { systemWithMemory } from "./memory.ts";
import { resetOrders } from "./orders.ts";
import { createSession, type SessionOptions } from "./session.ts";
import { keepLastTurns } from "./trim-history.ts";

const MESSAGES = [
  "Has my order 1041 been delivered yet?",
  "What time do you close?",
  "How much is delivery?",
  "How much is a salt coffee?",
  "Then please add a salt coffee to my order",
];

const WAYS: [string, SessionOptions][] = [
  ["send the full history", {}],
  ["send only the last 2 turns", { pickHistory: (history) => keepLastTurns(history, 2) }],
  ["send the last 2 turns, plus the remembered order", { pickHistory: (history) => keepLastTurns(history, 2), buildSystem: systemWithMemory }],
];

for (const [name, options] of WAYS) {
  resetOrders(); // every run starts from the same order data
  const session = createSession(options);
  let result = await session.receive(MESSAGES[0]);
  for (const message of MESSAGES.slice(1)) result = await session.receive(message);
  const first = result.calls[0];
  console.log(`--- ${name} ---`);
  console.log(`Customer (message 5): ${MESSAGES[4]}`);
  console.log(`  sent ${first.messageCount} messages, ${first.chars} characters`);
  for (const line of result.log) console.log(`  ${line}`);
  console.log(`Mộc: ${result.answer}\n`);
}

--- send the full history ---
Customer (message 5): Then please add a salt coffee to my order
  sent 11 messages, 1136 characters
  add_item({"order_id":"1041","item":"salt coffee","quantity":1}) → {"order_id":"1041","items":["milk coffee","egg banh mi","salt coffee"],"status":"being made"}
Mộc: Done! Order 1041 now has: milk coffee, egg banh mi, salt coffee, and it is still being made.

--- send only the last 2 turns ---
Customer (message 5): Then please add a salt coffee to my order
  sent 5 messages, 413 characters
Mộc: Could you send Mộc your order number or the phone number you ordered with, so we can add it?

--- send the last 2 turns, plus the remembered order ---
Customer (message 5): Then please add a salt coffee to my order
  sent 5 messages, 413 characters
  add_item({"order_id":"1041","item":"salt coffee","quantity":1}) → {"order_id":"1041","items":["milk coffee","egg banh mi","salt coffee"],"status":"being made"}
Mộc: Done! Order 1041 now has: milk coffee, egg banh mi, salt coffee, and it is still being made.